Skip to content
Python

fix(memhub): separate local diagnostics from safe exports

kristinhe7f23b74c367f
diff --git a/memhub/apps/cli/src/cli-logs.test.ts b/memhub/apps/cli/src/cli-logs.test.ts
index 63ba392..b8714f2 100644
--- a/memhub/apps/cli/src/cli-logs.test.ts
+++ b/memhub/apps/cli/src/cli-logs.test.ts
@@ -1,6 +1,7 @@
-import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
+import { inflateRawSync } from "node:zlib";
import { DiagnosticLogger } from "@one-memory/runtime-control";
import { afterEach, expect, it, vi } from "vitest";
@@ -29,6 +30,27 @@ async function fixture() {
return dataDir;
}
+function readZipEntry(archive: Buffer, expected: string): string {
+ let offset = 0;
+ while (archive.readUInt32LE(offset) === 0x04034b50) {
+ const method = archive.readUInt16LE(offset + 8);
+ const compressedSize = archive.readUInt32LE(offset + 18);
+ const nameLength = archive.readUInt16LE(offset + 26);
+ const extraLength = archive.readUInt16LE(offset + 28);
+ const nameStart = offset + 30;
+ const dataStart = nameStart + nameLength + extraLength;
+ const name = archive.subarray(nameStart, nameStart + nameLength).toString();
+ const compressed = archive.subarray(dataStart, dataStart + compressedSize);
+ if (name === expected) {
+ if (method === 0) return compressed.toString();
+ if (method === 8) return inflateRawSync(compressed).toString();
+ throw new Error("unsupported zip compression");
+ }
+ offset = dataStart + compressedSize;
+ }
+ throw new Error(`missing ${expected}`);
+}
+
it("prints matching diagnostic events as NDJSON without starting Runtime", async () => {
const dataDir = await fixture();
let stdout = "";
@@ -74,6 +96,25 @@ it("exports one support zip to the requested downloads directory", async () => {
});
expect(receipt.path).toMatch(/MemHub-日志-3800-\d{8}-\d{4}\.zip$/);
expect(await readdir(downloads)).toEqual([path.basename(receipt.path)]);
+ const archive = await readFile(receipt.path);
+ expect(JSON.parse(readZipEntry(archive, "summary.json")).commit).toBeNull();
+});
+
+it("fails closed for an explicitly supplied invalid commit", async () => {
+ const dataDir = await fixture();
+ const downloads = await mkdtemp(path.join(os.tmpdir(), "om-cli-downloads-"));
+ roots.push(downloads);
+ vi.spyOn(process.stdout, "write").mockImplementation(() => true);
+
+ await expect(
+ run(["logs", "export", "--json"], {
+ ONE_MEMORY_DATA_DIR: dataDir,
+ ONE_MEMORY_EXPORT_DIR: downloads,
+ ONE_MEMORY_BUILD_NUMBER: "3800",
+ ONE_MEMORY_COMMIT: "unknown",
+ }),
+ ).rejects.toThrow("diagnostic_commit_invalid");
+ expect(await readdir(downloads)).toEqual([]);
});
it("never emits a secret from a malformed manually written event", async () => {
diff --git a/memhub/apps/cli/src/cli.ts b/memhub/apps/cli/src/cli.ts
index 78d57be..443ff20 100644
--- a/memhub/apps/cli/src/cli.ts
+++ b/memhub/apps/cli/src/cli.ts
@@ -1048,15 +1048,21 @@ async function runCommand(
entry: "cli",
}));
const target = path.join(downloads, diagnosticZipFilename(build));
+ const environmentCommit = environment.ONE_MEMORY_COMMIT?.trim();
+ const embeddedCommit =
+ typeof __ONE_MEMORY_COMMIT__ === "string"
+ ? __ONE_MEMORY_COMMIT__.trim()
+ : "";
+ const commit =
+ environmentCommit ||
+ (embeddedCommit && embeddedCommit !== "unknown"
+ ? embeddedCommit
+ : undefined);
const receipt = await exportDiagnosticZip({
dataDir: config.dataDir,
target,
productVersion: build,
- commit:
- environment.ONE_MEMORY_COMMIT?.trim() ||
- (typeof __ONE_MEMORY_COMMIT__ === "string"
- ? __ONE_MEMORY_COMMIT__
- : "unknown"),
+ ...(commit === undefined ? {} : { commit }),
coreSha256: await sha256File(config.coreCommand).catch(() => undefined),
home: os.homedir(),
crashReportsDir: path.join(
diff --git a/memhub/apps/cli/src/log-report.test.ts b/memhub/apps/cli/src/log-report.test.ts
index 1b78b20..88e7575 100644
--- a/memhub/apps/cli/src/log-report.test.ts
+++ b/memhub/apps/cli/src/log-report.test.ts
@@ -1,4 +1,4 @@
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, rm, symlink } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { DiagnosticLogger } from "@one-memory/runtime-control";
@@ -41,11 +41,15 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
session_id: "11111111-1111-4111-8111-111111111111",
details: {
count: 1,
+ reason: "/Users/me/private-query.txt",
+ status: "sk-live-secret-token",
+ source_id: "credential.private",
prompt: "secret-token-123",
authorization: "Bearer secret-token-123",
} as unknown as Record<string, string | number | boolean | null>,
safe_stack: [
"packages/runtime-control/src/example.ts:42:7",
+ "packages/runtime/sk-live-secret-token.ts:1:1",
"/Users/me/private.py:1",
"C:\\Users\\me\\private.py:1",
],
@@ -56,6 +60,8 @@ it("redacts unsafe details and absolute stack frames from the read surface", ()
"packages/runtime-control/src/example.ts:42:7",
]);
expect(JSON.stringify(redacted)).not.toContain("secret-token-123");
+ expect(JSON.stringify(redacted)).not.toContain("sk-live-secret-token");
+ expect(JSON.stringify(redacted)).not.toContain("credential.private");
expect(JSON.stringify(redacted)).not.toContain("/Users/me");
expect(JSON.stringify(redacted)).not.toContain("C:\\\\Users");
});
@@ -132,3 +138,27 @@ it("follows a newly appended event without stopping Runtime", async () => {
});
expect(seen).toEqual(["service.ready", "core.failed"]);
});
+
+it("fails follow mode when the diagnostic directory is unsafe", async () => {
+ const dataDir = await mkdtemp(
+ path.join(os.tmpdir(), "om-cli-follow-unsafe-"),
+ );
+ const external = await mkdtemp(
+ path.join(os.tmpdir(), "om-cli-follow-external-"),
+ );
+ roots.push(dataDir, external);
+ await symlink(external, path.join(dataDir, "logs"));
+ const query = parseLogQuery(["--follow"]);
+ expect(query).not.toBeNull();
+ if (!query) throw new Error("query_missing");
+ await expect(
+ followLogs({
+ dataDir,
+ query,
+ signal: new AbortController().signal,
+ maxEvents: 1,
+ emit: () => undefined,
+ wait: async () => undefined,
+ }),
+ ).rejects.toThrow("log_directory_unreadable");
+});
diff --git a/memhub/apps/cli/src/log-report.ts b/memhub/apps/cli/src/log-report.ts
index 02c3e55..d2d73a8 100644
--- a/memhub/apps/cli/src/log-report.ts
+++ b/memhub/apps/cli/src/log-report.ts
@@ -3,6 +3,7 @@ import path from "node:path";
import type { DiagnosticEventV1 } from "@one-memory/runtime-control";
import {
diagnosticLogFiles,
+ projectDiagnosticEventForSafeOutput,
readDiagnosticEvents,
} from "@one-memory/runtime-control";
@@ -24,55 +25,6 @@ export interface LogQuery {
const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
const NAME = /^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$/;
-const SAFE_DETAIL_KEYS = new Set([
- "action",
- "attempt",
- "count",
- "method",
- "mode",
- "phase",
- "platform",
- "reason",
- "result_count",
- "source_id",
- "spawned",
- "status",
- "version",
- "parent_session_id",
- "invalid_lines",
- "truncated",
- "runtime_state",
- "core_state",
- "broker_state",
- "instance_id",
- "open_file_count",
- "memory_bytes",
- "thread_count",
- "database_error_count",
- "file_count",
- "sqlite_file_count",
- "byte_file_count",
- "total_bytes",
- "copied_bytes",
- "bytes_per_second",
- "stored_count",
- "revised_count",
- "skipped_count",
- "error_count",
- "restart_streak",
- "pending_count",
- "prepared_count",
- "queue_depth",
- "in_flight_count",
- "call_count",
- "created_count",
- "updated_count",
- "superseded_count",
- "deleted_count",
- "operation_kind",
- "file_kind",
-]);
-
export function parseLogQuery(arguments_: readonly string[]): LogQuery | null {
if (arguments_.filter((item) => item === "--json").length > 1) return null;
const values: Partial<LogQuery> = { errorsOnly: false, follow: false };
@@ -238,63 +190,13 @@ export function renderLogText(
return `${lines.join("\n")}\n`;
}
-/** Defence in depth for the CLI read surface. `DiagnosticLogger` already
- * allow-lists keys, but a future writer, a manually edited log, or a malformed
- * line must not turn `memhub logs` into an exfiltration path. */
+/** Defence in depth for the CLI read surface. Local writers accept arbitrary
+ * bounded scalar keys, so every outward read surface must reuse the same
+ * field-and-value projection as the support-bundle exporter. */
export function redactDiagnosticEvent(
event: DiagnosticEventV1,
): DiagnosticEventV1 {
- const redacted: DiagnosticEventV1 = {
- schema_version: 1,
- timestamp: event.timestamp,
- level: event.level,
- service: event.service,
- component: event.component,
- event: event.event,
- outcome: event.outcome,
- session_id: event.session_id,
- ...(event.trace_id === undefined ? {} : { trace_id: event.trace_id }),
- ...(event.incident_id === undefined
- ? {}
- : { incident_id: event.incident_id }),
- ...(event.instance_id === undefined
- ? {}
- : { instance_id: event.instance_id }),
- ...(event.operation_id === undefined
- ? {}
- : { operation_id: event.operation_id }),
- ...(event.request_id === undefined ? {} : { request_id: event.request_id }),
- ...(event.job_id === undefined ? {} : { job_id: event.job_id }),
- ...(event.task_id === undefined ? {} : { task_id: event.task_id }),
- ...(event.duration_ms === undefined
- ? {}
- : { duration_ms: event.duration_ms }),
- ...(event.state_before === undefined
- ? {}
- : { state_before: event.state_before }),
- ...(event.state_after === undefined
- ? {}
- : { state_after: event.state_after }),
- ...(event.error_code === undefined ? {} : { error_code: event.error_code }),
- ...(event.error_type === undefined ? {} : { error_type: event.error_type }),
- ...(event.safe_stack === undefined
- ? {}
- : {
- safe_stack: event.safe_stack.filter(
- (frame) =>
- !frame.startsWith("/") &&
- !/^[A-Za-z]:\\/.test(frame) &&
- !frame.includes(".."),
- ),
- }),
- };
- const details = Object.fromEntries(
- Object.entries(event.details ?? {}).filter(([key]) =>
- SAFE_DETAIL_KEYS.has(key),
- ),
- );
- if (Object.keys(details).length > 0) redacted.details = details;
- return redacted;
+ return projectDiagnosticEventForSafeOutput(event);
}
export async function followLogs(options: {
@@ -316,6 +218,7 @@ export async function followLogs(options: {
...options.query,
tail: 5_000,
});
+ if (report.failure) throw new Error(report.failure);
for (const event of report.events) {
const identity = JSON.stringify(event);
if (seen.has(identity)) continue;
diff --git a/memhub/apps/core/src/one_memory/api.py b/memhub/apps/core/src/one_memory/api.py
index 80fa413..f528213 100644
--- a/memhub/apps/core/src/one_memory/api.py
+++ b/memhub/apps/core/src/one_memory/api.py
@@ -998,9 +998,7 @@ def create_app(
app.state.recall_target = GovernedRecallTarget(
app.state.memory_governance_store,
filter_memories=app.state.source_read_access.memories,
- semantic_search=lambda query: _sdk_memory_runtime(
- app
- ).semantic_recall(query),
+ semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
# Read per request rather than cached: an import that lands
# while the app is open should make its workspace reachable
# without a restart. Affordable only because the read is served
@@ -1102,9 +1100,7 @@ def create_app(
authorize_generation=lambda profile, question, grounding: (
_authorize_answer_generation(app, profile, question, grounding)
),
- semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(
- query
- ),
+ semantic_search=lambda query: _sdk_memory_runtime(app).semantic_recall(query),
resolve_workspace=lambda workspace_path: resolve_workspace_scopes(
workspace_path,
evidence_store.known_workspaces(user_id="local"),
@@ -1128,21 +1124,21 @@ def create_app(
profiles=app.state.model_profile_store,
secrets=app.state.secret_store,
data_dir=runtime_settings.data_dir,
- transport_factory=lambda profile: lambda: AuditedModelTransport(
- app.state.model_request_audit_store,
- profile_id=profile.profile_id,
- profile_revision=profile.revision,
- provider_kind=str(profile.provider_kind),
- model=profile.model,
- inner=(
- BudgetedModelTransport(model_request_budget)
- if model_request_budget is not None
- else None
- ),
+ transport_factory=lambda profile: (
+ lambda: AuditedModelTransport(
+ app.state.model_request_audit_store,
+ profile_id=profile.profile_id,
+ profile_revision=profile.revision,
+ provider_kind=str(profile.provider_kind),
+ model=profile.model,
+ inner=(
+ BudgetedModelTransport(model_request_budget)
+ if model_request_budget is not None
+ else None
+ ),
+ )
),
- sedimentation_enabled=lambda: _config_store(
- app
- ).sedimentation_enabled(),
+ sedimentation_enabled=lambda: _config_store(app).sedimentation_enabled(),
)
app.state.sdk_memory_runtime = SdkMemoryRuntime(
coordinator=app.state.sdk_write_coordinator,
@@ -1167,10 +1163,8 @@ def create_app(
exclude_error_codes=NON_RETRYABLE_POST_WRITE_CODES,
)
for job in unknown:
- receipt = (
- await app.state.sdk_write_coordinator.inquire_write_outcome(
- job.request_id
- )
+ receipt = await app.state.sdk_write_coordinator.inquire_write_outcome(
+ job.request_id
)
if receipt is None:
continue
@@ -1610,9 +1604,7 @@ def create_app(
)
if getattr(
app.state, "memory_authority", MemoryAuthority.LEGACY
- ) is MemoryAuthority.SDK and request.url.path.startswith(
- legacy_paths
- ):
+ ) is MemoryAuthority.SDK and request.url.path.startswith(legacy_paths):
return JSONResponse(
status_code=409,
content={"detail": "sdk_memory_authority_required"},
@@ -1734,7 +1726,7 @@ def create_app(
trace_id=trace_id,
instance_id=runtime_settings.instance_id,
duration_ms=elapsed_ms(started),
- details={**diagnostic_details, "status": str(response.status_code)},
+ details={**diagnostic_details, "status_code": response.status_code},
)
return response
except Exception as error:
@@ -3783,9 +3775,7 @@ def create_app(
all_memories, projected, nodes, edges = await asyncio.to_thread(read_governed_part)
try:
hy_graph = (
- None
- if revoked
- else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
+ None if revoked else await _sdk_memory_runtime(app).graph_snapshot(limit=limit)
)
except HySdkError:
hy_graph = None
@@ -3894,9 +3884,7 @@ def create_app(
active_queue, in_flight_count = supervisor.active_depths()
planned, current = supervisor.manual_batch_progress()
sdk_counts = (
- supervisor.status_counts()
- if app.state.memory_authority is MemoryAuthority.SDK
- else {}
+ supervisor.status_counts() if app.state.memory_authority is MemoryAuthority.SDK else {}
)
# `running` counts rows the store still marks running, which includes
# work a worker has already given up on. Reporting that as "what is
@@ -3917,9 +3905,7 @@ def create_app(
),
recent_completions=depths.recent_completions,
running=sdk_counts.get("running", running_rows),
- stranded_running=(
- 0 if sdk_counts else max(0, running_rows - in_flight_count)
- ),
+ stranded_running=(0 if sdk_counts else max(0, running_rows - in_flight_count)),
unexecuted_policy_failures=store.unexecuted_policy_failures(),
waiting_for_model=sdk_counts.get(
"waiting_for_model",
@@ -5286,7 +5272,7 @@ def _report_visibility_failure(phase: str, error: BaseException) -> None:
event="memory_visibility.background_failed",
outcome="failed",
error_type=type(error).__name__,
- details={"phase": phase, "reason": str(error)[:200]},
+ details={"phase": phase},
)
diff --git a/memhub/apps/core/src/one_memory/diagnostic_log.py b/memhub/apps/core/src/one_memory/diagnostic_log.py
index 6e9c402..2f5a2c7 100644
--- a/memhub/apps/core/src/one_memory/diagnostic_log.py
+++ b/memhub/apps/core/src/one_memory/diagnostic_log.py
@@ -20,8 +20,10 @@ from __future__ import annotations
import atexit
import json
import logging
+import math
import os
import re
+import stat
import sys
import threading
import time
@@ -72,80 +74,30 @@ class _Refused(OSError):
"""The path is not one we are willing to write to, and never will be."""
-DiagnosticLevel = Literal["info", "warn", "error"]
+DiagnosticLevel = Literal["debug", "info", "warn", "error"]
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
_EVENT = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$")
-_DETAIL_KEYS = frozenset(
- {
- "action",
- "attempt",
- "count",
- "method",
- "mode",
- "phase",
- "platform",
- "reason",
- "result_count",
- "source_id",
- "spawned",
- "status",
- "version",
- "parent_session_id",
- "invalid_lines",
- "truncated",
- "open_file_count",
- "memory_bytes",
- "thread_count",
- "database_error_count",
- "route_path",
- "sqlite_errorcode",
- "sqlite_errorname",
- "file_count",
- "sqlite_file_count",
- "byte_file_count",
- "total_bytes",
- "copied_bytes",
- "bytes_per_second",
- "stored_count",
- "revised_count",
- "skipped_count",
- "error_count",
- "pending_count",
- "prepared_count",
- "queue_depth",
- "in_flight_count",
- "call_count",
- "created_count",
- "updated_count",
- "superseded_count",
- "deleted_count",
- "operation_kind",
- "index_complete",
- "lexical_hits",
- "semantic_hits",
- "generation",
- "timed_out",
- "failure_site",
- "error_class",
- "file_kind",
- "file_size",
- "file_mtime_ns",
- "file_inode",
- "cursor_new",
- "cursor_old",
- "cursor_read_to_end",
- "cursor_rewind_branch",
- "summed_ms",
- "model_ms",
- "accounted_ms",
- "unaccounted_ms",
- "conversation_hash",
- # The sedimentation switch: a bounded boolean, the same shape as
- # `spawned` and `index_complete`. Left out of this list it did not
- # drop a log line, it turned the switch's own request into a 500.
- "enabled",
- }
+_DIAGNOSTIC_LOG_NAME = re.compile(
+ r"^diagnostic-(runtime|core|host-bridge|desktop|cli)\.jsonl(?:\.\d+)?$"
+)
+_DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
+_CODE_FRAME = re.compile(
+ r"^(?:(?:apps|packages|scripts)/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:\d+"
+ r"|one_memory/(?:[A-Za-z0-9_-]+/)*[A-Za-z0-9_.-]+:\d+:[A-Za-z_][A-Za-z0-9_]*"
+ r"|[A-Za-z0-9_][A-Za-z0-9_.-]*\.py:\d+:[A-Za-z_][A-Za-z0-9_]*)$"
+)
+_SHORT_CREDENTIAL_TOKEN = re.compile(
+ r"(?<![A-Za-z0-9])([A-Za-z0-9]{24,47})(?:[_-])?(?![A-Za-z0-9])"
)
+_REFUSED_DETAIL_KEYS = frozenset({"__proto__", "prototype", "constructor"})
+_MAX_DETAIL_KEYS = 32
+_MAX_SAFE_INTEGER = (1 << 53) - 1
+_EXPORT_SECRET_MARKER = re.compile(
+ r"(?:^|[._:-])(?:authorization|credential|password|prompt|query|secret|token)(?:$|[._:-])",
+ re.IGNORECASE,
+)
+_TRACE_STRING_DETAIL_KEYS = frozenset({"phase", "mode", "status", "reason", "failure_site"})
+_TRACE_NUMBER_DETAIL_KEYS = frozenset({"count"})
_trace_id: ContextVar[str | None] = ContextVar("one_memory_trace_id", default=None)
_active: DiagnosticLog | None = None
_database_errors: Counter[tuple[str, str]] = Counter()
@@ -264,7 +216,12 @@ class DiagnosticLog:
}
value.update({key: _identifier(item) for key, item in identifiers.items() if item})
if duration_ms is not None:
- if not isinstance(duration_ms, int) or duration_ms < 0:
+ if (
+ not isinstance(duration_ms, int)
+ or isinstance(duration_ms, bool)
+ or duration_ms < 0
+ or duration_ms > _MAX_SAFE_INTEGER
+ ):
raise ValueError("diagnostic_duration_invalid")
value["duration_ms"] = duration_ms
for field_name, item in {
@@ -276,7 +233,9 @@ class DiagnosticLog:
if item is not None:
value[field_name] = _identifier(item)
if safe_stack:
- value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack[:256]]
+ if len(safe_stack) > 16:
+ raise ValueError("diagnostic_stack_invalid")
+ value["safe_stack"] = [_safe_stack_line(line) for line in safe_stack]
if details:
value["details"] = _details(details)
self._submit(_serialize(value))
@@ -289,6 +248,8 @@ class DiagnosticLog:
event: str,
outcome: str,
) -> dict[str, object]:
+ if level not in ("debug", "info", "warn", "error"):
+ raise ValueError("diagnostic_level_invalid")
return {
"schema_version": 1,
"timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
@@ -651,24 +612,17 @@ def _drain_diagnostics_at_exit() -> None:
def diagnostic_event(**values: object) -> None:
- active = _active
- if active is None:
- return
- if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
- return
try:
+ active = _active
+ if active is None:
+ return
+ if values.get("level") == "debug" and not logging.getLogger().isEnabledFor(logging.DEBUG):
+ return
active.write(**values) # type: ignore[arg-type]
- except OSError:
- # Diagnostics must not become a new product outage when a disk is full
- # or the log directory is temporarily unavailable.
- return
- except ValueError:
- # Same promise, same reason, for a rejected payload instead of a failed
- # write. An unlisted `details` key once turned every external recall
- # into a 500 and then killed the process, and the sedimentation switch
- # into a permanent failure. The payload itself is still wrong and is
- # still caught in CI by tests/test_diagnostic_payload_keys.py; what
- # must not happen is the caller paying for it.
+ except Exception:
+ # Diagnostics are best-effort: validation, serialization, logging, and
+ # filesystem failures must never become a product outage. BaseException
+ # deliberately remains visible so process-control signals still work.
return
@@ -691,46 +645,95 @@ def read_trace_events(
if (request_id is None) == (trace_id is None):
raise ValueError("exactly_one_trace_identity_required")
key, expected = ("request_id", request_id) if request_id is not None else ("trace_id", trace_id)
- if expected is None or not _IDENTIFIER.fullmatch(expected):
+ if expected is None or not _export_identifier(expected):
raise ValueError("invalid_trace_identity")
selected: list[dict[str, object]] = []
- allowed_details = {"phase", "mode", "status", "reason", "failure_site", "count"}
- files = sorted(
- (data_dir / "logs").glob("diagnostic-*.jsonl*"),
- key=lambda path: path.stat().st_mtime,
- )
- for path in files:
- if path.is_symlink() or not path.is_file():
+ directory = data_dir / "logs"
+ try:
+ directory_info = directory.lstat()
+ except FileNotFoundError:
+ return ()
+ if stat.S_ISLNK(directory_info.st_mode) or not stat.S_ISDIR(directory_info.st_mode):
+ raise ValueError("diagnostic_directory_unsafe")
+ files: list[tuple[Path, float]] = []
+ for path in directory.iterdir():
+ if not _DIAGNOSTIC_LOG_NAME.fullmatch(path.name):
+ continue
+ try:
+ info = path.lstat()
+ except OSError:
continue
- for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+ if stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode):
+ files.append((path, info.st_mtime))
+ for path, _mtime in sorted(files, key=lambda item: item[1]):
+ for line in _read_regular_file_no_follow(path).splitlines():
try:
event = json.loads(line)
except (TypeError, json.JSONDecodeError):
continue
if not isinstance(event, dict) or event.get(key) != expected:
continue
- details = event.get("details")
- projected = {
- name: event[name]
- for name in (
- "timestamp",
- "component",
- "event",
- "outcome",
- "duration_ms",
- "error_code",
- "error_type",
- )
- if name in event
+ timestamp = event.get("timestamp")
+ if not isinstance(timestamp, str):
+ continue
+ try:
+ datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+ except ValueError:
+ continue
+ component = event.get("component")
+ event_name = event.get("event")
+ outcome = event.get("outcome")
+ projected: dict[str, object] = {
+ "timestamp": timestamp,
+ "component": (
+ component
+ if isinstance(component, str) and _export_identifier(component)
+ else "diagnostic.redacted"
+ ),
+ "event": (
+ event_name
+ if isinstance(event_name, str) and _export_identifier(event_name)
+ else "diagnostic.redacted"
+ ),
+ "outcome": (
+ outcome
+ if isinstance(outcome, str) and _export_identifier(outcome)
+ else "redacted"
+ ),
}
+ duration_ms = event.get("duration_ms")
+ if (
+ isinstance(duration_ms, int)
+ and not isinstance(duration_ms, bool)
+ and 0 <= duration_ms <= _MAX_SAFE_INTEGER
+ ):
+ projected["duration_ms"] = duration_ms
+ for name in ("error_code", "error_type"):
+ value = event.get(name)
+ if isinstance(value, str) and _export_identifier(value):
+ projected[name] = value
+ details = event.get("details")
if isinstance(details, dict):
- projected["details"] = {
- name: value for name, value in details.items() if name in allowed_details
- }
+ safe_details = _trace_details(details)
+ if safe_details:
+ projected["details"] = safe_details
selected.append(projected)
return tuple(selected[-max(1, min(limit, 500)) :])
+def _read_regular_file_no_follow(path: Path) -> str:
+ descriptor = os.open(path, os.O_RDONLY | _O_NOFOLLOW)
+ try:
+ if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+ raise ValueError("diagnostic_file_unsafe")
+ with os.fdopen(descriptor, encoding="utf-8", errors="replace") as handle:
+ descriptor = -1
+ return handle.read()
+ finally:
+ if descriptor >= 0:
+ os.close(descriptor)
+
+
def current_trace_id() -> str | None:
return _trace_id.get()
@@ -837,8 +840,77 @@ def diagnostic_exception(error: BaseException) -> dict[str, object]:
}
+def _contains_credential_material(value: str) -> bool:
+ return bool(
+ re.search(
+ r"(?:^|[^A-Za-z0-9])(?:sk-|sk_|omhg_|ghp_|github_pat_)[A-Za-z0-9._-]+",
+ value,
+ re.IGNORECASE,
+ )
+ or re.search(r"(?:^|[^A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]+", value, re.IGNORECASE)
+ or re.search(r"(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])", value)
+ or re.search(r"(?:^|[^A-Za-z0-9])AKID[A-Za-z0-9]{12,}", value, re.IGNORECASE)
+ or re.search(r"(?:^|[^A-Za-z0-9])AIza[A-Za-z0-9_-]{20,}", value)
+ or re.search(r"(?:^|[^A-Za-z0-9])GOCSPX-[A-Za-z0-9_-]+", value)
+ or any(
+ _is_conservative_short_credential(match.group(1))
+ for match in _SHORT_CREDENTIAL_TOKEN.finditer(value)
+ )
+ or re.search(
+ r"\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b",
+ value,
+ )
+ or any(
+ len(candidate) >= 48
+ and any(character.isalpha() for character in candidate)
+ and any(character.isdigit() for character in candidate)
+ for candidate in re.findall(r"\b[A-Za-z0-9._-]{48,}\b", value)
+ )
+ )
+
+
+def _is_conservative_short_credential(value: str) -> bool:
+ return bool(
+ re.search(r"[A-Z]", value)
+ and re.search(r"[a-z]", value)
+ and re.search(r"\d", value)
+ and not re.fullmatch(r"[A-Fa-f0-9]+", value)
+ )
+
+
+def _export_identifier(value: str) -> bool:
+ return bool(
+ _IDENTIFIER.fullmatch(value)
+ and not _EXPORT_SECRET_MARKER.search(value)
+ and not _contains_credential_material(value)
+ )
+
+
+def _trace_details(values: Mapping[object, object]) -> dict[str, str | int | float]:
+ result: dict[str, str | int | float] = {}
+ for key, value in values.items():
+ if (
+ key in _TRACE_STRING_DETAIL_KEYS
+ and isinstance(value, str)
+ and _export_identifier(value)
+ ):
+ result[str(key)] = value
+ elif key in _TRACE_NUMBER_DETAIL_KEYS and isinstance(value, (int, float)):
+ if isinstance(value, bool):
+ continue
+ safe_integer = isinstance(value, int) and abs(value) <= _MAX_SAFE_INTEGER
+ safe_float = (
+ isinstance(value, float)
+ and math.isfinite(value)
+ and (not value.is_integer() or abs(value) <= _MAX_SAFE_INTEGER)
+ )
+ if safe_integer or safe_float:
+ result[str(key)] = value
+ return result
+
+
def _serialize(value: Mapping[str, object]) -> str:
- return json.dumps(value, ensure_ascii=True, separators=(",", ":")) + "\n"
+ return json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(",", ":")) + "\n"
def _uuid(value: str | None) -> str | None:
@@ -864,24 +936,47 @@ def _event_name(value: str) -> str:
def _safe_stack_line(value: str) -> str:
- if not value or len(value) > 256 or "\n" in value or "\r" in value:
- raise ValueError("diagnostic_stack_invalid")
- if Path(value.split(":", 1)[0]).is_absolute() or "file://" in value:
+ if (
+ not value
+ or len(value) > 256
+ or any(character in value for character in "\0\r\n")
+ or _contains_credential_material(value)
+ ):
raise ValueError("diagnostic_stack_invalid")
- return value
+ if _CODE_FRAME.fullmatch(value):
+ return value
+ if value.startswith("thread|"):
+ frames = value.removeprefix("thread|").split(" > ")
+ if frames and all(_CODE_FRAME.fullmatch(frame) for frame in frames):
+ return value
+ raise ValueError("diagnostic_stack_invalid")
def _details(
values: Mapping[str, str | int | float | bool | None],
) -> dict[str, str | int | float | bool | None]:
+ if len(values) > _MAX_DETAIL_KEYS:
+ raise ValueError("diagnostic_detail_count_invalid")
result: dict[str, str | int | float | bool | None] = {}
for key, value in values.items():
- if key not in _DETAIL_KEYS:
+ if (
+ not isinstance(key, str)
+ or not _DETAIL_KEY.fullmatch(key)
+ or key in _REFUSED_DETAIL_KEYS
+ ):
raise ValueError("diagnostic_detail_key_invalid")
if isinstance(value, str):
- if not value or len(value) > 256 or "\n" in value or "\r" in value:
+ if not value or len(value) > 256 or any(character in value for character in "\0\r\n"):
+ raise ValueError("diagnostic_detail_value_invalid")
+ elif isinstance(value, float):
+ if not math.isfinite(value) or (value.is_integer() and abs(value) > _MAX_SAFE_INTEGER):
+ raise ValueError("diagnostic_detail_value_invalid")
+ elif isinstance(value, bool) or value is None:
+ pass
+ elif isinstance(value, int):
+ if abs(value) > _MAX_SAFE_INTEGER:
raise ValueError("diagnostic_detail_value_invalid")
- elif value is not None and not isinstance(value, (int, float, bool)):
+ else:
raise ValueError("diagnostic_detail_value_invalid")
result[key] = value
return result
diff --git a/memhub/apps/core/src/one_memory/exception_reporting.py b/memhub/apps/core/src/one_memory/exception_reporting.py
index d544801..9a1da5b 100644
--- a/memhub/apps/core/src/one_memory/exception_reporting.py
+++ b/memhub/apps/core/src/one_memory/exception_reporting.py
@@ -3,8 +3,8 @@
Every site that catches `Exception` broadly must call `report_exception`
before (or instead of) continuing, so a failure never looks like success
and a stack is always on disk. Content (conversation text, keys, SQL
-parameters) never reaches the log: only exception type, a truncated
-message, and file:line:function frames.
+parameters, and exception messages) never reaches the diagnostic log: only
+exception type, a repeat count, and file:line:function frames.
"""
from __future__ import annotations
@@ -67,7 +67,7 @@ def report_exception(
event=event,
outcome="failed",
**payload,
- details={"reason": _safe_message(error), "count": _RECENT[key]},
+ details={"count": _RECENT[key]},
)
if full:
_logger.error(
diff --git a/memhub/apps/core/src/one_memory/imports/live_sync.py b/memhub/apps/core/src/one_memory/imports/live_sync.py
index 7ce56f4..d432add 100644
--- a/memhub/apps/core/src/one_memory/imports/live_sync.py
+++ b/memhub/apps/core/src/one_memory/imports/live_sync.py
@@ -557,12 +557,10 @@ class PollingLiveSyncSupervisor:
)
def _note_internal_error(self, source_id: str, error: Exception) -> None:
- # Type alone used to be all this said. It is not enough to act on:
- # `OperationalError` covers a locked database, a missing table and a
- # corrupt page alike, and without the message and the frames two dozen
- # of these in a log could only be guessed at. The stack is the
- # redacted, `one_memory`-relative one, so it names the call site
- # without printing anyone's paths or query text.
+ # The ordinary local log keeps the exception message for debugging.
+ # The structured diagnostic event deliberately keeps only the stable
+ # error code, source id, type, and relative stack: exception messages
+ # can contain paths, query text, or credentials and are exportable data.
logging.getLogger(__name__).warning(
"live_sync_internal_error %s: %s", type(error).__name__, error
)
@@ -572,10 +570,7 @@ class PollingLiveSyncSupervisor:
event="live_sync.internal_error",
outcome="failed",
error_code="live_sync_internal_error",
- # `reason` is the listed key that can carry the message; see
- # `diagnostic_log._DETAIL_KEYS`. Truncated because a detail value
- # longer than 256 characters is rejected.
- details={"source_id": source_id, "reason": str(error)[:200]},
+ details={"source_id": source_id},
**diagnostic_exception(error),
)
if isinstance(error, sqlite3.Error):
diff --git a/memhub/apps/core/src/one_memory/imports/runner.py b/memhub/apps/core/src/one_memory/imports/runner.py
index 1a068a1..e5d60b4 100644
--- a/memhub/apps/core/src/one_memory/imports/runner.py
+++ b/memhub/apps/core/src/one_memory/imports/runner.py
@@ -867,10 +867,7 @@ class ImportRunner:
outcome="failed",
job_id=job_id,
error_code="import_heartbeat_failed",
- # `reason` is the listed key for this; see
- # `diagnostic_log._DETAIL_KEYS`. Truncated: a detail value
- # over 256 characters is rejected.
- details={"reason": str(error)[:200]},
+ details={"phase": "heartbeat"},
**diagnostic_exception(error),
)
diff --git a/memhub/apps/core/src/one_memory/loop_watchdog.py b/memhub/apps/core/src/one_memory/loop_watchdog.py
index a1a9ea2..e294152 100644
--- a/memhub/apps/core/src/one_memory/loop_watchdog.py
+++ b/memhub/apps/core/src/one_memory/loop_watchdog.py
@@ -14,6 +14,7 @@ stuck in it.
"""
import logging
+import re
import sys
import threading
import time
@@ -23,6 +24,7 @@ from types import FrameType
logger = logging.getLogger("one_memory.loop_watchdog")
DEFAULT_STALL_SECONDS = 2.0
+_SAFE_FALLBACK_FILE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$")
class LoopWatchdog:
@@ -120,22 +122,28 @@ def _safe_stack_for_main_thread() -> tuple[str, ...]:
def _safe_stacks_for_all_threads() -> tuple[str, ...]:
- """Return one content-free, bounded stack summary for every live thread."""
- names = {thread.ident: thread.name for thread in threading.enumerate()}
+ """Return bounded stack summaries without recording thread names."""
lines: list[str] = []
- for ident, frame in sorted(sys._current_frames().items()):
+ for _ident, frame in sorted(sys._current_frames().items()):
stack: list[str] = []
- # Most specific frame first so the 256-character privacy bound keeps
- # the likely culprit rather than only the thread bootstrap frames.
+ # Most specific frame first; append only complete frames so truncation
+ # can never turn a valid location into free text.
for entry in reversed(traceback.extract_stack(frame)[-8:]):
parts = entry.filename.replace("\\", "/").split("/")
try:
root = parts.index("one_memory")
relative = "/".join(parts[root:])
except ValueError:
- relative = parts[-1]
+ fallback = parts[-1]
+ relative = fallback if _SAFE_FALLBACK_FILE.fullmatch(fallback) else "unknown.py"
name = entry.name if entry.name.isidentifier() else "call"
- stack.append(f"{relative}:{entry.lineno}:{name}")
- thread_name = names.get(ident, "unknown").replace("\n", " ").replace("\r", " ")
- lines.append(f"thread={thread_name[:48]}|{' > '.join(stack)}"[:256])
+ candidate = f"{relative}:{entry.lineno}:{name}"
+ separator = "" if not stack else " > "
+ if len("thread|") + len(" > ".join(stack)) + len(separator) + len(candidate) > 256:
+ break
+ stack.append(candidate)
+ if stack:
+ lines.append(f"thread|{' > '.join(stack)}")
+ if len(lines) == 16:
+ break
return tuple(lines)
diff --git a/memhub/apps/core/tests/test_diagnostic_log.py b/memhub/apps/core/tests/test_diagnostic_log.py
index 558bab1..ca46fff 100644
--- a/memhub/apps/core/tests/test_diagnostic_log.py
+++ b/memhub/apps/core/tests/test_diagnostic_log.py
@@ -18,11 +18,13 @@ from one_memory.diagnostic_log import (
diagnostic_exception,
diagnostics_enabled,
diagnostics_requested,
+ read_trace_events,
record_database_error,
reset_diagnostics,
reset_trace_id,
resource_snapshot,
)
+from one_memory.exception_reporting import report_exception
def test_debug_events_are_collected_only_at_debug_level(tmp_path, monkeypatch) -> None:
@@ -105,7 +107,31 @@ def test_structured_log_is_private_correlated_and_owner_only(tmp_path):
assert log.path.stat().st_mode & 0o777 == 0o600
-def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
+def test_arbitrary_valid_detail_keys_and_scalars_are_recorded(tmp_path):
+ log = DiagnosticLog(tmp_path)
+ details = {
+ "Custom.metric-v1": "ready",
+ "noneValue": None,
+ "boolValue": False,
+ "intValue": 7,
+ "floatValue": 1.5,
+ }
+ log.write(
+ level="info",
+ component="core.lifecycle",
+ event="core.started",
+ outcome="succeeded",
+ details=details,
+ )
+ assert log.flush()
+ assert json.loads(log.path.read_text())["details"] == details
+
+
+ "key",
+ ["__proto__", "prototype", "constructor", "_private", "9startsWithDigit", "a" * 65],
+)
+def test_dangerous_or_malformed_detail_keys_are_rejected(tmp_path, key):
log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_detail_key_invalid"):
log.write(
@@ -113,8 +139,76 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
component="hy.bridge",
event="hy.failed",
outcome="failed",
- details={"prompt": "private"},
+ details={key: "value"},
+ )
+
+
[email protected]("value", [{}, [], object()])
+def test_complex_detail_values_are_rejected(tmp_path, value):
+ log = DiagnosticLog(tmp_path)
+ with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ details={"payload": value},
)
+
+
[email protected]("value", [float("nan"), float("inf"), float("-inf")])
+def test_non_finite_detail_values_are_rejected(tmp_path, value):
+ log = DiagnosticLog(tmp_path)
+ with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ details={"metric": value},
+ )
+
+
[email protected]("value", [2**53, -(2**53), float(2**53), -float(2**53)])
+def test_numeric_detail_values_outside_json_safe_range_are_rejected(tmp_path, value):
+ log = DiagnosticLog(tmp_path)
+ with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ details={"count": value},
+ )
+
+
[email protected]("value", ["", "nul\0byte", "carriage\rreturn", "line\nfeed", "x" * 257])
+def test_invalid_detail_strings_are_rejected(tmp_path, value):
+ log = DiagnosticLog(tmp_path)
+ with pytest.raises(ValueError, match="diagnostic_detail_value_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ details={"message": value},
+ )
+
+
+def test_more_than_32_detail_keys_are_rejected(tmp_path):
+ log = DiagnosticLog(tmp_path)
+ with pytest.raises(ValueError, match="diagnostic_detail_count_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ details={f"key{index}": index for index in range(33)},
+ )
+
+
+def test_absolute_or_credential_bearing_stack_is_rejected(tmp_path):
+ log = DiagnosticLog(tmp_path)
with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
log.write(
level="error",
@@ -123,16 +217,90 @@ def test_arbitrary_detail_and_absolute_stack_are_rejected(tmp_path):
outcome="failed",
safe_stack=("/Users/me/private.py:1",),
)
+ with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ safe_stack=("one_memory/sk-live-secret-token.py:1:call",),
+ )
+
+
+def test_safe_stack_uses_strict_shared_grammar_and_sixteen_line_limit(tmp_path):
+ log = DiagnosticLog(tmp_path)
+ for line in (
+ "arbitrary stack text",
+ "thread=worker|threading.py:12:wait",
+ "FILE:///Users/me/private.py:1:call",
+ "one_memory/worker.py:9:call /Users/private",
+ "one_memory/GOCSPX-privateCredential.py:9:call",
+ "packages/ASIA1234567890ABCDEF/file.ts:1:1",
+ "packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
+ ):
+ with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ safe_stack=(line,),
+ )
+
+ accepted = (
+ "apps/desktop/src/main.ts:80:1",
+ "one_memory/diagnostic_log.py:895:_safe_stack_line",
+ "threading.py:1045:_bootstrap_inner",
+ "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+ )
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ safe_stack=accepted,
+ )
+ with pytest.raises(ValueError, match="diagnostic_stack_invalid"):
+ log.write(
+ level="error",
+ component="hy.bridge",
+ event="hy.failed",
+ outcome="failed",
+ safe_stack=tuple(f"one_memory/worker.py:{index + 1}:call" for index in range(17)),
+ )
+ assert log.flush()
+ assert json.loads(log.path.read_text())["safe_stack"] == list(accepted)
+
+
+def test_invalid_level_is_strict_but_diagnostic_event_remains_fail_open(
+ tmp_path, monkeypatch
+) -> None:
+ log = DiagnosticLog(tmp_path)
+ with pytest.raises(ValueError, match="diagnostic_level_invalid"):
+ log.write(
+ level="fatal", # type: ignore[arg-type]
+ component="core.lifecycle",
+ event="core.failed",
+ outcome="failed",
+ )
+ monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
+ receipt = configure_diagnostics(tmp_path / "fail-open")
+ try:
+ diagnostic_event(
+ level="fatal",
+ component="core.lifecycle",
+ event="core.failed",
+ outcome="failed",
+ )
+ finally:
+ reset_diagnostics()
+ assert not receipt.path.exists()
-def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) -> None:
- """Rejection protects the log; it must not become the caller's outage.
- Two of our own nodes shipped with unlisted `details` keys. Both sat on a
- request path, so the rejection surfaced as a 500 to the person using the
- app — including the sedimentation switch, whose every request failed
- because logging the toggle raised before the endpoint could answer.
- """
+def test_a_rejected_payload_does_not_break_the_caller_or_later_events(
+ tmp_path, monkeypatch
+) -> None:
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
receipt = configure_diagnostics(tmp_path)
try:
@@ -141,20 +309,122 @@ def test_a_rejected_payload_does_not_break_the_caller(tmp_path, monkeypatch) ->
component="memory.sedimentation",
event="memory_sedimentation.toggled",
outcome="succeeded",
- details={"prompt": "private"},
+ details={"payload": ["not", "scalar"]},
+ )
+ diagnostic_event(
+ level="info",
+ component="memory.sedimentation",
+ event="memory_sedimentation.toggled",
+ outcome="succeeded",
+ details={"custom.metric": 1},
)
finally:
reset_diagnostics()
- # Rejected, so nothing is written — but the call returned.
- assert not receipt.path.exists() or receipt.path.read_text() == ""
+ events = [json.loads(line) for line in receipt.path.read_text().splitlines()]
+ assert [event["details"] for event in events] == [{"custom.metric": 1}]
-def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
- """`enabled` is a bounded boolean and has to be a legal detail key.
+def test_trace_projection_does_not_expose_arbitrary_local_detail_keys(tmp_path):
+ log = DiagnosticLog(tmp_path)
+ log.write(
+ level="info",
+ component="core.api",
+ event="memory.request.completed",
+ outcome="succeeded",
+ request_id="request-1",
+ details={
+ "phase": "recall",
+ "reason": "/Users/private/query.txt",
+ "status": "sk-live-secret-token",
+ "custom.private": "local-only",
+ },
+ )
+ assert log.flush()
+ events = read_trace_events(tmp_path, request_id="request-1")
+ assert events[0]["details"] == {"phase": "recall"}
- Omitting it from the allow-list is what made the switch's own endpoint
- answer 500 while still saving the new value.
- """
+
+def test_trace_projection_rejects_symlinked_directory_and_redacts_top_level_secrets(
+ tmp_path,
+) -> None:
+ request_id = "request-1"
+ logs = tmp_path / "logs"
+ logs.mkdir()
+ event = {
+ "timestamp": "2026-10-23T01:00:00Z",
+ "request_id": request_id,
+ "component": "secret.token",
+ "event": "credential.failed",
+ "outcome": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+ "error_code": "GOCSPX-clientSecret_A1",
+ "error_type": "ASIA1234567890ABCDEF",
+ "details": {
+ "phase": "recall",
+ "reason": "x_ASIA1234567890ABCDEF",
+ "mode": "x_GOCSPX-clientSecret_A1",
+ "status": "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+ "failure_site": "AIzaSyExampleCredential1234567890",
+ "count": 1,
+ },
+ }
+ (logs / "diagnostic-core.jsonl").write_text(json.dumps(event) + "\n")
+ projected = read_trace_events(tmp_path, request_id=request_id)
+ assert projected == (
+ {
+ "timestamp": "2026-10-23T01:00:00Z",
+ "component": "diagnostic.redacted",
+ "event": "diagnostic.redacted",
+ "outcome": "redacted",
+ "details": {"phase": "recall", "count": 1},
+ },
+ )
+
+ external = tmp_path / "external"
+ external.mkdir()
+ (logs / "diagnostic-core.jsonl").unlink()
+ logs.rmdir()
+ logs.symlink_to(external, target_is_directory=True)
+ with pytest.raises(ValueError, match="diagnostic_directory_unsafe"):
+ read_trace_events(tmp_path, request_id=request_id)
+
+
+def test_credential_detection_is_conservative_and_matches_export_rules() -> None:
+ for value in (
+ "ASIA1234567890ABCDEF",
+ "GOCSPX-clientSecret_A1",
+ "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+ "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8_",
+ ):
+ assert diagnostic_log._contains_credential_material(value)
+ for value in (
+ "11111111-1111-4111-8111-111111111111",
+ "abcdef1234567890abcdef1234567890abcdef12",
+ "cursor",
+ ):
+ assert not diagnostic_log._contains_credential_material(value)
+
+
+def test_trace_reader_never_follows_a_diagnostic_file_symlink(tmp_path) -> None:
+ logs = tmp_path / "logs"
+ logs.mkdir()
+ outside = tmp_path / "outside.jsonl"
+ outside.write_text(
+ json.dumps(
+ {
+ "timestamp": "2026-10-23T01:00:00Z",
+ "request_id": "request-1",
+ "component": "core.api",
+ "event": "memory.request.completed",
+ "outcome": "succeeded",
+ }
+ )
+ + "\n"
+ )
+ (logs / "diagnostic-core.jsonl").symlink_to(outside)
+ assert read_trace_events(tmp_path, request_id="request-1") == ()
+
+
+def test_the_sedimentation_toggle_payload_is_recorded(tmp_path, monkeypatch) -> None:
monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
receipt = configure_diagnostics(tmp_path)
try:
@@ -181,7 +451,7 @@ def test_exception_diagnostic_keeps_code_location_not_message():
component="hy.bridge",
event="hy.failed",
outcome="failed",
- details={"prompt": secret},
+ details={"prompt": f"{secret}\n"},
)
except ValueError as error:
safe = diagnostic_exception(error)
@@ -191,6 +461,28 @@ def test_exception_diagnostic_keeps_code_location_not_message():
assert str(__file__) not in json.dumps(safe)
+def test_report_exception_never_writes_the_exception_message_to_diagnostics(
+ tmp_path, monkeypatch
+) -> None:
+ monkeypatch.setenv("ONE_MEMORY_DIAGNOSTICS", "1")
+ receipt = configure_diagnostics(tmp_path)
+ secret = "sk-live-private-provider-response"
+ try:
+ report_exception(
+ "core.test",
+ "core.test.failed",
+ RuntimeError(f"provider failed with {secret} at /Users/private/query.txt"),
+ )
+ finally:
+ reset_diagnostics()
+ payload = receipt.path.read_text()
+ assert secret not in payload
+ assert "/Users/private" not in payload
+ event = json.loads(payload)
+ assert event["details"] == {"count": 1}
+ assert event["error_type"] == "RuntimeError"
+
+
def test_old_logs_are_removed_but_symlinks_are_never_followed(tmp_path):
log = DiagnosticLog(tmp_path)
log.path.parent.mkdir(parents=True)
diff --git a/memhub/apps/core/tests/test_diagnostic_log_off_loop.py b/memhub/apps/core/tests/test_diagnostic_log_off_loop.py
index 5461770..7874ebd 100644
--- a/memhub/apps/core/tests/test_diagnostic_log_off_loop.py
+++ b/memhub/apps/core/tests/test_diagnostic_log_off_loop.py
@@ -449,7 +449,7 @@ def test_invalid_input_is_still_rejected_on_the_caller_side(tmp_path):
component="hy.bridge",
event="hy.failed",
outcome="failed",
- details={"prompt": "private"},
+ details={"invalid key": "private"},
)
log.close()
diff --git a/memhub/apps/core/tests/test_diagnostic_payload_keys.py b/memhub/apps/core/tests/test_diagnostic_payload_keys.py
index 21e4a0a..f247410 100644
--- a/memhub/apps/core/tests/test_diagnostic_payload_keys.py
+++ b/memhub/apps/core/tests/test_diagnostic_payload_keys.py
@@ -1,22 +1,57 @@
"""An instrument must never be able to break the thing it measures.
-`diagnostic_event` validates its payload: an unlisted `details` key, a missing
-`level`, or a detail value over 256 characters all raise. Two of our own nodes
-shipped with unlisted keys, and because they sat on a request path and on the
-loop heartbeat, they turned every external recall into a 500 and then killed
-the process. Both were caught only by running the built runtime, not by tests.
+`diagnostic_event` validates its payload: a malformed `details` key, a missing
+`level`, or an unsafe detail value is rejected. A diagnostic payload must never
+be able to break the request path or loop heartbeat it is intended to measure.
So this checks the payloads at their source, by walking every literal call.
"""
import ast
+import re
from pathlib import Path
import pytest
-from one_memory.diagnostic_log import _DETAIL_KEYS
-
ROOT = Path(__file__).resolve().parent.parent / "src" / "one_memory"
+DETAIL_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
+REFUSED_DETAIL_KEYS = {"__proto__", "prototype", "constructor"}
+EXPORTED_STRING_DETAIL_KEYS = {
+ "action",
+ "method",
+ "mode",
+ "phase",
+ "platform",
+ "reason",
+ "source_id",
+ "status",
+ "version",
+ "parent_session_id",
+ "runtime_state",
+ "core_state",
+ "broker_state",
+ "instance_id",
+ "operation_kind",
+ "file_kind",
+ "last_probe",
+ "signal",
+}
+PRIVATE_DETAIL_KEYS = {
+ "api_key",
+ "authorization",
+ "content",
+ "credential",
+ "memory",
+ "memory_text",
+ "message",
+ "messages",
+ "password",
+ "prompt",
+ "query",
+ "raw_text",
+ "secret",
+ "token",
+}
#: Every module, not a hand-picked list. The two payloads that caused real
#: outages were in files nobody thought to add here, so the only shape of this
@@ -33,6 +68,19 @@ def _literal_calls(path: Path):
yield node
+def _contains_free_text_conversion(node: ast.AST) -> bool:
+ for item in ast.walk(node):
+ if (
+ isinstance(item, ast.Call)
+ and isinstance(item.func, ast.Name)
+ and item.func.id in {"str", "repr", "_safe_message"}
+ ):
+ return True
+ if isinstance(item, ast.Attribute) and item.attr in {"args", "message"}:
+ return True
+ return False
+
+
@pytest.mark.parametrize("relative", SOURCES)
def test_every_diagnostic_payload_is_valid(relative: str) -> None:
path = ROOT / relative
@@ -45,7 +93,22 @@ def test_every_diagnostic_payload_is_valid(relative: str) -> None:
for keyword in node.keywords:
if keyword.arg != "details" or not isinstance(keyword.value, ast.Dict):
continue
- for key in keyword.value.keys:
- if isinstance(key, ast.Constant) and key.value not in _DETAIL_KEYS:
- problems.append(f"{relative}:{node.lineno}: unlisted details key {key.value!r}")
+ for key, expression in zip(keyword.value.keys, keyword.value.values, strict=True):
+ if not isinstance(key, ast.Constant):
+ continue
+ value = key.value
+ if (
+ not isinstance(value, str)
+ or not DETAIL_KEY.fullmatch(value)
+ or value in REFUSED_DETAIL_KEYS
+ ):
+ problems.append(f"{relative}:{node.lineno}: invalid details key {value!r}")
+ elif value.lower() in PRIVATE_DETAIL_KEYS:
+ problems.append(f"{relative}:{node.lineno}: private details key {value!r}")
+ elif value in EXPORTED_STRING_DETAIL_KEYS and _contains_free_text_conversion(
+ expression
+ ):
+ problems.append(
+ f"{relative}:{node.lineno}: exported detail {value!r} uses free text"
+ )
assert not problems, "\n".join(problems)
diff --git a/memhub/apps/core/tests/test_loop_watchdog.py b/memhub/apps/core/tests/test_loop_watchdog.py
index 329ee77..e2c3402 100644
--- a/memhub/apps/core/tests/test_loop_watchdog.py
+++ b/memhub/apps/core/tests/test_loop_watchdog.py
@@ -102,6 +102,8 @@ def test_stall_captures_a_safe_stack_for_every_thread() -> None:
assert watchdog.stall_stacks
captured = watchdog.stall_stacks[0]
- assert any(line.startswith("thread=MainThread|") for line in captured)
- assert any(line.startswith("thread=source-reader-test|") for line in captured)
+ assert captured
+ assert len(captured) <= 16
+ assert all(line.startswith("thread|") for line in captured)
+ assert all("source-reader-test" not in line and "MainThread" not in line for line in captured)
assert all("/Users/" not in line and "\n" not in line for line in captured)
diff --git a/memhub/apps/desktop/src/diagnostics-ipc.test.ts b/memhub/apps/desktop/src/diagnostics-ipc.test.ts
index 10cd249..bc3d7c4 100644
--- a/memhub/apps/desktop/src/diagnostics-ipc.test.ts
+++ b/memhub/apps/desktop/src/diagnostics-ipc.test.ts
@@ -94,12 +94,14 @@ it("returns only diagnostic events written during this desktop launch", async ()
await mkdir(logs, { recursive: true });
const target = path.join(logs, "diagnostic-core.jsonl");
const old = {
+ schema_version: 1,
timestamp: "2026-10-14T09:00:00.000Z",
level: "info",
service: "core",
component: "core.startup",
event: "core.old",
outcome: "succeeded",
+ session_id: "11111111-1111-4111-8111-111111111111",
};
await writeFile(target, `${JSON.stringify(old)}\n`);
const cursor = await captureDebugLogCursor(dataDir);
@@ -109,6 +111,11 @@ it("returns only diagnostic events written during this desktop launch", async ()
timestamp: "2026-10-14T10:01:00.000Z",
event: "core.current",
level: "warn",
+ details: {
+ count: 1,
+ "custom.private": "/Users/private/query.txt",
+ credential_hint: "ghp_private_token",
+ },
};
await appendFile(target, `${JSON.stringify(current)}\nnot-json\n`);
@@ -118,8 +125,11 @@ it("returns only diagnostic events written during this desktop launch", async ()
expect(snapshot.entries[0]).toMatchObject({
event: "core.current",
level: "warn",
+ details: { count: 1 },
source: "diagnostic",
});
+ expect(JSON.stringify(snapshot)).not.toContain("/Users/private");
+ expect(JSON.stringify(snapshot)).not.toContain("ghp_private_token");
});
it("tracks diagnostic JSONL across rotation without rereading the old prefix", async () => {
@@ -130,12 +140,14 @@ it("tracks diagnostic JSONL across rotation without rereading the old prefix", a
const target = path.join(logs, "diagnostic-core.jsonl");
const event = (name: string, timestamp: string) =>
JSON.stringify({
+ schema_version: 1,
timestamp,
level: "info",
service: "core",
component: "core.test",
event: name,
outcome: "succeeded",
+ session_id: "22222222-2222-4222-8222-222222222222",
});
await writeFile(target, `${event("before", "2026-10-14T09:00:00.000Z")}\n`);
const cursor = await captureDebugLogCursor(dataDir);
diff --git a/memhub/apps/desktop/src/diagnostics-ipc.ts b/memhub/apps/desktop/src/diagnostics-ipc.ts
index 8c5af92..9791b46 100644
--- a/memhub/apps/desktop/src/diagnostics-ipc.ts
+++ b/memhub/apps/desktop/src/diagnostics-ipc.ts
@@ -1,10 +1,13 @@
import { lstat, open, readdir } from "node:fs/promises";
import path from "node:path";
import {
+ type DiagnosticEventV1,
type DiagnosticLogger,
+ diagnosticEvent,
diagnosticZipFilename,
exportDiagnosticZip,
inspectBackgroundHealth,
+ projectDiagnosticEventForSafeOutput,
recordDiagnosticMarker,
} from "@one-memory/runtime-control";
import type { BrowserWindow, IpcMainInvokeEvent } from "electron";
@@ -370,42 +373,58 @@ function debugLogEntry(
const timestamp = typeof value.timestamp === "string" ? value.timestamp : "";
const timestampMs = Date.parse(timestamp);
if (!Number.isFinite(timestampMs) || timestampMs < startedAt) return null;
- const level = value.level;
- const service = value.service;
- const component = value.component;
- const event = value.event;
- const outcome = value.outcome;
- if (
- !["info", "warn", "error"].includes(String(level)) ||
- !["runtime", "core", "host-bridge", "desktop", "cli"].includes(
- String(service),
- ) ||
- typeof component !== "string" ||
- typeof event !== "string" ||
- typeof outcome !== "string"
- ) {
+ let projected: DiagnosticEventV1;
+ try {
+ projected = projectDiagnosticEventForSafeOutput(
+ diagnosticEvent({
+ timestamp,
+ level: value.level as never,
+ service: value.service as never,
+ component: value.component as never,
+ event: value.event as never,
+ outcome: value.outcome as never,
+ session_id: value.session_id as never,
+ ...(typeof value.trace_id === "string"
+ ? { trace_id: value.trace_id }
+ : {}),
+ ...(typeof value.duration_ms === "number"
+ ? { duration_ms: value.duration_ms }
+ : {}),
+ ...(typeof value.error_code === "string"
+ ? { error_code: value.error_code }
+ : {}),
+ ...(typeof value.error_type === "string"
+ ? { error_type: value.error_type }
+ : {}),
+ ...(value.details && typeof value.details === "object"
+ ? { details: value.details as never }
+ : {}),
+ }),
+ );
+ } catch {
return null;
}
- const details = safeDebugDetails(value.details);
return {
id: `${filename}:${lineNumber}:${timestamp}`,
- timestamp,
- level: level as DebugLogEntry["level"],
- service: service as DebugLogEntry["service"],
- component,
- event,
- outcome,
- ...(typeof value.duration_ms === "number"
- ? { duration_ms: value.duration_ms }
- : {}),
- ...(typeof value.error_code === "string"
- ? { error_code: value.error_code }
- : {}),
- ...(typeof value.error_type === "string"
- ? { error_type: value.error_type }
- : {}),
- ...(typeof value.trace_id === "string" ? { trace_id: value.trace_id } : {}),
- ...(details === undefined ? {} : { details }),
+ timestamp: projected.timestamp,
+ level: projected.level,
+ service: projected.service,
+ component: projected.component,
+ event: projected.event,
+ outcome: projected.outcome,
+ ...(projected.duration_ms === undefined
+ ? {}
+ : { duration_ms: projected.duration_ms }),
+ ...(projected.error_code === undefined
+ ? {}
+ : { error_code: projected.error_code }),
+ ...(projected.error_type === undefined
+ ? {}
+ : { error_type: projected.error_type }),
+ ...(projected.trace_id === undefined
+ ? {}
+ : { trace_id: projected.trace_id }),
+ ...(projected.details === undefined ? {} : { details: projected.details }),
source: "diagnostic",
};
}
@@ -523,25 +542,6 @@ function logLineTimestamp(
return new Date(Math.max(startedAt, fallback)).toISOString();
}
-function safeDebugDetails(
- value: unknown,
-): Record<string, string | number | boolean | null> | undefined {
- if (!value || typeof value !== "object" || Array.isArray(value))
- return undefined;
- const details: Record<string, string | number | boolean | null> = {};
- for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
- if (
- item === null ||
- typeof item === "string" ||
- typeof item === "number" ||
- typeof item === "boolean"
- ) {
- details[key] = item;
- }
- }
- return Object.keys(details).length ? details : undefined;
-}
-
async function pipelineLogFiles(root: string): Promise<PipelineLogCursor[]> {
const rootInfo = await lstat(root).catch(() => null);
if (!rootInfo?.isDirectory() || rootInfo.isSymbolicLink()) return [];
diff --git a/memhub/docs/PRODUCT_LOG_TROUBLESHOOTING.md b/memhub/docs/PRODUCT_LOG_TROUBLESHOOTING.md
index 89a253a..94af606 100644
--- a/memhub/docs/PRODUCT_LOG_TROUBLESHOOTING.md
+++ b/memhub/docs/PRODUCT_LOG_TROUBLESHOOTING.md
@@ -6,10 +6,13 @@
2. 记忆沉淀有没有持续运行、产出了多少条;
3. 如果停住,停在导入、模型调用还是最终写入。
-默认 `info` 下的结构化诊断日志不包含会话正文、模型输入输出或 API Key。`debug`
-会额外启用 HY pipeline trace,其中可能包含完整 prompt/response 和获授权的对话正文。
-排查时优先通过 App 的“导出日志”提供文件,不要直接发送整个 `~/.one-memory` 数据目录;
-debug 期间产生的 pipeline 日志在分享前必须由开发人员确认并脱敏。
+默认 `info` 下的官方结构化事件不应写入会话正文、模型输入输出或 API Key;但本地原始日志允许
+新增受限标量标签,仍应按私人数据处理,不能直接对外发送。`debug` 会额外启用 HY pipeline
+trace,其中可能包含完整 prompt/response 和获授权的对话正文。
+
+排查时只通过 App 的“导出日志”提供文件:导出过程会按安全白名单重新生成诊断事件,不会打包
+原始日志。不要直接发送 `diagnostic-*.jsonl`、整个 `~/.one-memory` 数据目录或 debug pipeline
+日志;后者如确需分享,必须由开发人员另行确认并脱敏。
## 1. Log level 设什么
@@ -39,7 +42,8 @@ debug 可能把完整模型 prompt/response 写入本机调试日志,不得长
## 2. 日志在哪里
App 内推荐直接使用“运行诊断 → 导出日志”。开发人员远程指导时,优先使用
-`memhub logs`,它会再次过滤不安全字段。需要现场查看原始结构化日志时,Core 日志默认位于:
+`memhub logs`,它会再次过滤不安全字段。原始结构化日志只用于本机现场查看,不得作为附件发送。
+Core 日志默认位于:
```text
~/.one-memory/logs/diagnostic-core.jsonl
diff --git a/memhub/docs/contracts/diagnostic-logging.md b/memhub/docs/contracts/diagnostic-logging.md
index 4a9ef56..ef410d2 100644
--- a/memhub/docs/contracts/diagnostic-logging.md
+++ b/memhub/docs/contracts/diagnostic-logging.md
@@ -15,9 +15,17 @@ Desktop -> Runtime -> Core -> HY -> Host 的实际执行链。日志不是用户
可选关联字段为 `trace_id`、`incident_id`、`instance_id`、`operation_id`、`request_id`、
`job_id`、`task_id`、`duration_ms`、状态、稳定错误码、错误类型和清理后的仓库相对栈帧。
-`details` 只接受代码声明的标量白名单。禁止记录对话原文、查询、回答、模型输入输出、
-请求体、响应体、token、API Key、secret ref、完整用户路径或异常 message/cause。未知字段和
-不合规标识拒绝写入。日志写入失败不能阻断产品主链。
+`details` 的本地写入合同只约束结构,不维护业务字段白名单:最多 32 个 key;key 必须是
+1–64 字符的字母、数字、点、下划线或连字符并以字母开头;value 只接受 `null`、布尔值、
+跨 Python/JavaScript 可无损表示的有限数字(整数限于 ±(2^53-1))或不超过 256 字符且不含
+控制换行的短字符串,不接受对象和数组。这样新增诊断标签
+不需要先修改中央白名单。诊断日志的校验、序列化、排队或落盘发生任何异常时都只能丢弃该条
+日志,不能阻断产品主链。
+
+本地写入放宽不代表允许官方代码记录用户内容。对话原文、查询、回答、模型输入输出、请求体、
+响应体、token、API Key、secret ref、完整用户路径和异常 message/cause 仍禁止进入诊断事件,
+由类型、静态检查和隐私测试共同防止误用;确需保存完整用户输入的调试能力必须使用独立的本地
+trace/session 文件,不得混入可导出的诊断日志。
## 关联规则
@@ -34,9 +42,16 @@ Desktop -> Runtime -> Core -> HY -> Host 的实际执行链。日志不是用户
host-bridge、desktop 和 cli。单文件 4 MiB,保留 3 份轮转;额外清理 14 天以前的文件并将
总量限制在 128 MiB。POSIX 目录为 0700、文件为 0600。
-读取器只枚举上述固定文件名的 regular file,不跟随符号链接,不接受任意路径。损坏行跳过
-并计数,不让一行损坏遮蔽其余证据。诊断导出是单个脱敏 JSON,包含版本、平台、架构、
-非秘密状态和最多 5000 条已验证事件,不包含数据库或配置文件。
+读取器只枚举上述固定文件名的 regular file,不跟随符号链接,不接受任意路径。日常本地查询
+遇到损坏行时跳过并计数,不让一行损坏遮蔽其余证据。
+
+导出是独立的安全边界:它重新读取并校验全部候选事件,只把明确列入导出白名单的字段投影为
+新的 `diagnostics/events.jsonl`;details 不只检查 key,还按字段约束 value 类型,并拒绝自由文本、
+路径和凭证形态的字符串。绝不复制原始 `diagnostic-*.jsonl`、普通运行日志或崩溃报告。
+导出包还可以包含固定 schema 的系统摘要和问题时间点,但不接受调用方传入的任意对象展开。
+任何事件、marker、安全投影或日志目录读取失败时,导出必须 fail-closed:删除临时文件、不发布
+诊断包,更不能回退为打包原始日志。发布使用已设好权限的临时文件原子创建最终目标,已存在的
+目标不得覆盖,发布后也不存在再 chmod 失败留下伪成功文件的窗口。
## 覆盖与节流
@@ -56,7 +71,8 @@ host-bridge、desktop 和 cli。单文件 4 MiB,保留 3 份轮转;额外清
## 验收
-隐私测试必须先植入一条可检测的秘密作为正向对照,再证明真实链路日志和导出均不含该秘密。
-至少用隔离数据制造 Core 启动失败、HY 失败和宿主连接失败,并仅凭日志还原失败组件、时间、
+隐私测试必须先向本地日志植入带未知 key 的可检测秘密作为正向对照,再证明本地读取仍保留该
+证据、导出安全投影必定剔除它;另需证明损坏行会阻止导出且不会留下伪成功文件。至少用隔离
+数据制造 Core 启动失败、HY 失败和宿主连接失败,并仅凭允许导出的字段还原失败组件、时间、
关联 ID、稳定错误码和代码位置。自动测试通过只能标记 verified,正式安装 App 的真实体验仍需
单独验收。
diff --git a/memhub/package.json b/memhub/package.json
index cc2bc33..0fc8e9b 100644
--- a/memhub/package.json
+++ b/memhub/package.json
@@ -2,7 +2,7 @@
"name": "one-memory",
"version": "1.0.5",
"release": {
- "build_number": 15,
+ "build_number": 16,
"macos_min_version": "13.0",
"app_update": "manual",
"runtime_manifest_schema": 1,
diff --git a/memhub/packages/runtime-control/src/diagnostic-log.test.ts b/memhub/packages/runtime-control/src/diagnostic-log.test.ts
index a6cbb92..2373847 100644
--- a/memhub/packages/runtime-control/src/diagnostic-log.test.ts
+++ b/memhub/packages/runtime-control/src/diagnostic-log.test.ts
@@ -1,4 +1,5 @@
import {
+ chmod,
lstat,
mkdir,
mkdtemp,
@@ -21,8 +22,10 @@ import {
diagnosticLogFiles,
exportDiagnosticBundle,
exportDiagnosticZip,
+ projectDiagnosticEventForSafeOutput,
pruneDiagnosticLogs,
readDiagnosticEvents,
+ recordDiagnosticMarker,
writeDiagnosticEventSync,
} from "./diagnostic-log.js";
@@ -66,6 +69,30 @@ describe("diagnostic log", () => {
});
});
+ it("keeps the synchronous writer fail-open for invalid input", async () => {
+ const dataDir = await root();
+ expect(() =>
+ writeDiagnosticEventSync(
+ {
+ dataDir,
+ service: "desktop",
+ sessionId: "55555555-5555-4555-8555-555555555555",
+ },
+ {
+ level: "error",
+ component: "desktop.lifecycle",
+ event: "desktop.early_start_failed",
+ outcome: "failed",
+ details: { nested: ["private"] } as never,
+ },
+ ),
+ ).not.toThrow();
+ expect(await readDiagnosticEvents(dataDir)).toEqual({
+ events: [],
+ invalid_lines: 0,
+ });
+ });
+
it("writes structured, private events and reads them by correlation identity", async () => {
const dataDir = await root();
const session = "11111111-1111-4111-8111-111111111111";
@@ -89,6 +116,8 @@ describe("diagnostic log", () => {
open_file_count: 7,
memory_bytes: 4096,
thread_count: 4,
+ "cache.hit-rate": 0.75,
+ custom_label: "warm",
},
});
const result = await readDiagnosticEvents(dataDir, { traceId: trace });
@@ -101,6 +130,8 @@ describe("diagnostic log", () => {
event: "runtime.ready",
details: expect.objectContaining({
open_file_count: 7,
+ "cache.hit-rate": 0.75,
+ custom_label: "warm",
}),
});
const file = (await diagnosticLogFiles(dataDir))[0];
@@ -184,7 +215,7 @@ describe("diagnostic log", () => {
}
});
- it("rejects arbitrary detail fields and absolute stack paths", () => {
+ it("strictly rejects complex detail values, dangerous keys and absolute stacks", () => {
const base = {
timestamp: new Date().toISOString(),
level: "error" as const,
@@ -195,11 +226,111 @@ describe("diagnostic log", () => {
session_id: "11111111-1111-4111-8111-111111111111",
};
expect(() =>
- diagnosticEvent({ ...base, details: { prompt: "private" } }),
+ diagnosticEvent({
+ ...base,
+ details: { nested: { secret: "private" } } as never,
+ }),
+ ).toThrow("diagnostic_detail_value_invalid");
+ expect(() =>
+ diagnosticEvent({ ...base, details: { constructor: "private" } }),
).toThrow("diagnostic_detail_key_invalid");
+ expect(() =>
+ diagnosticEvent({ ...base, details: { "9invalid": "private" } }),
+ ).toThrow("diagnostic_detail_key_invalid");
+ expect(() =>
+ diagnosticEvent({
+ ...base,
+ details: Object.fromEntries(
+ Array.from({ length: 33 }, (_, index) => [`key_${index}`, index]),
+ ),
+ }),
+ ).toThrow("diagnostic_detail_count_invalid");
+ expect(() =>
+ diagnosticEvent({ ...base, details: { message: "line1\nline2" } }),
+ ).toThrow("diagnostic_text_invalid");
+ expect(() =>
+ diagnosticEvent({ ...base, details: { metric: Number.NaN } }),
+ ).toThrow("diagnostic_detail_value_invalid");
+ expect(() =>
+ diagnosticEvent({
+ ...base,
+ details: { count: Number.MAX_SAFE_INTEGER + 1 },
+ }),
+ ).toThrow("diagnostic_detail_value_invalid");
+ expect(() =>
+ diagnosticEvent({ ...base, details: { list: ["private"] } as never }),
+ ).toThrow("diagnostic_detail_value_invalid");
expect(() =>
diagnosticEvent({ ...base, safe_stack: ["/Users/me/private.py:1"] }),
).toThrow("diagnostic_stack_invalid");
+ expect(() =>
+ diagnosticEvent({
+ ...base,
+ safe_stack: ["packages/runtime/sk-live-secret-token.ts:1:1"],
+ }),
+ ).toThrow("diagnostic_stack_invalid");
+ for (const line of [
+ "arbitrary stack text",
+ "thread=worker|threading.py:12:wait",
+ "FILE:///Users/me/private.py:1:call",
+ "one_memory/worker.py:9:call /Users/private",
+ "one_memory/GOCSPX-privateCredential.py:9:call",
+ "packages/ASIA1234567890ABCDEF/file.ts:1:1",
+ "packages/Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8/file.ts:1:1",
+ ]) {
+ expect(() => diagnosticEvent({ ...base, safe_stack: [line] })).toThrow(
+ "diagnostic_stack_invalid",
+ );
+ }
+ expect(() =>
+ diagnosticEvent({
+ ...base,
+ safe_stack: Array.from(
+ { length: 17 },
+ (_, index) => `one_memory/worker.py:${index + 1}:call`,
+ ),
+ }),
+ ).toThrow("diagnostic_stack_invalid");
+
+ const accepted = diagnosticEvent({
+ ...base,
+ safe_stack: [
+ "apps/desktop/src/main.ts:80:1",
+ "one_memory/diagnostic_log.py:895:_safe_stack_line",
+ "threading.py:1045:_bootstrap_inner",
+ "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+ ],
+ });
+ expect(accepted.safe_stack).toHaveLength(4);
+
+ const projected = projectDiagnosticEventForSafeOutput({
+ ...accepted,
+ safe_stack: [
+ "free text",
+ "thread=worker|one_memory/worker.py:9:call",
+ "packages/runtime-control/src/diagnostic-log.ts:42:7",
+ "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+ ],
+ });
+ expect(projected.safe_stack).toEqual([
+ "packages/runtime-control/src/diagnostic-log.ts:42:7",
+ "thread|threading.py:1045:_bootstrap_inner > one_memory/worker.py:9:call",
+ ]);
+ const conservative = projectDiagnosticEventForSafeOutput(
+ diagnosticEvent({
+ ...base,
+ details: {
+ source_id: "cursor",
+ reason: "abcdef1234567890abcdef1234567890abcdef12",
+ status: "11111111-1111-4111-8111-111111111111",
+ },
+ }),
+ );
+ expect(conservative.details).toEqual({
+ source_id: "cursor",
+ reason: "abcdef1234567890abcdef1234567890abcdef12",
+ status: "11111111-1111-4111-8111-111111111111",
+ });
});
it("drops malformed logger events without interrupting later writes", async () => {
@@ -216,7 +347,7 @@ describe("diagnostic log", () => {
component: "desktop.sdk_migration",
event: "sdk_migration.startup",
outcome: "succeeded",
- details: { state: "cutover" },
+ details: { nested: { secret: "private" } } as never,
}),
).resolves.toBeUndefined();
await logger.log({
@@ -224,13 +355,13 @@ describe("diagnostic log", () => {
component: "desktop.sdk_migration",
event: "sdk_migration.startup",
outcome: "succeeded",
- details: { runtime_state: "cutover" },
+ details: { arbitrary_state: "cutover" },
});
const result = await readDiagnosticEvents(dataDir);
expect(result.invalid_lines).toBe(0);
expect(result.events).toHaveLength(1);
- expect(result.events[0].details).toEqual({ runtime_state: "cutover" });
+ expect(result.events[0].details).toEqual({ arbitrary_state: "cutover" });
});
it("keeps code locations but strips exception messages and absolute prefixes", () => {
@@ -246,29 +377,162 @@ describe("diagnostic log", () => {
expect(JSON.stringify(safe)).not.toContain("/Users/me");
});
- it("marks incidents and exports only validated events", async () => {
+ it("exports events through an independent detail whitelist", async () => {
const dataDir = await root();
const logger = new DiagnosticLogger({ dataDir, service: "desktop" });
const incident = await logger.markIncident({
component: "desktop.diagnostics",
status: "degraded",
});
- await writeFile(
- path.join(dataDir, "logs", "diagnostic-core.jsonl"),
- "not-json\n",
- { mode: 0o600 },
- );
+ await logger.log({
+ level: "info",
+ component: "desktop.diagnostics",
+ event: "diagnostics.checked",
+ outcome: "succeeded",
+ details: {
+ count: 1,
+ reason: "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signaturevalue",
+ status: "ASIA1234567890ABCDEF",
+ source_id: "GOCSPX-clientSecret_A1",
+ mode: "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+ action: "AIzaSyExampleCredential1234567890",
+ platform: "x_ASIA1234567890ABCDEF",
+ phase: "x_GOCSPX-clientSecret_A1",
+ local_only_secret: "must-not-export",
+ },
+ });
+ await logger.log({
+ level: "error",
+ component: "credential.private",
+ event: "token.failed",
+ outcome: "ghp_private_token",
+ trace_id: "ghp_private_token",
+ instance_id: "ASIA1234567890ABCDEF",
+ request_id: "GOCSPX-clientSecret_A1",
+ task_id: "Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+ error_code: "secret.token",
+ });
const target = path.join(await root(), "bundle.json");
const result = await exportDiagnosticBundle({
dataDir,
target,
productVersion: "1.2.3",
- status: { status: "degraded" },
+ status: { status: "degraded", local_only_secret: "must-not-export" },
});
- expect(result).toEqual({ event_count: 1, invalid_lines: 1 });
+ expect(result).toEqual({ event_count: 3, invalid_lines: 0 });
const payload = JSON.parse(await readFile(target, "utf8"));
expect(payload.events[0].incident_id).toBe(incident.incident_id);
- expect(JSON.stringify(payload)).not.toContain(dataDir);
+ expect(payload.events[1].details).toEqual({ count: 1 });
+ expect(payload.events[2]).toMatchObject({
+ component: "diagnostic.redacted",
+ event: "diagnostic.redacted",
+ outcome: "redacted",
+ });
+ expect(payload.events[2].trace_id).toBeUndefined();
+ expect(payload.events[2].instance_id).toBeUndefined();
+ expect(payload.events[2].request_id).toBeUndefined();
+ expect(payload.events[2].task_id).toBeUndefined();
+ expect(payload.events[2].error_code).toBeUndefined();
+ expect(JSON.stringify(payload)).not.toContain("local_only_secret");
+ expect(JSON.stringify(payload)).not.toContain("must-not-export");
+ expect(JSON.stringify(payload)).not.toContain("eyJhbGciOiJIUzI1NiJ9");
+ expect(JSON.stringify(payload)).not.toContain("ASIA1234567890ABCDEF");
+ expect(JSON.stringify(payload)).not.toContain("GOCSPX-clientSecret_A1");
+ expect(JSON.stringify(payload)).not.toContain("x_ASIA1234567890ABCDEF");
+ expect(JSON.stringify(payload)).not.toContain("x_GOCSPX-clientSecret_A1");
+ expect(JSON.stringify(payload)).not.toContain("Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8");
+ expect(JSON.stringify(payload)).not.toContain(
+ "AIzaSyExampleCredential1234567890",
+ );
+ expect(JSON.stringify(payload)).not.toContain("ghp_private_token");
+ });
+
+ it("fails closed when exported build identity has a credential shape", async () => {
+ const dataDir = await root();
+ const outputDir = await root();
+ await expect(
+ exportDiagnosticBundle({
+ dataDir,
+ target: path.join(outputDir, "bundle.json"),
+ productVersion: "xoxb-123456-abcdef",
+ }),
+ ).rejects.toThrow("diagnostic_product_version_invalid");
+ await expect(
+ exportDiagnosticZip({
+ dataDir,
+ target: path.join(outputDir, "commit.zip"),
+ productVersion: "1.2.3",
+ commit: "ghp_private_token",
+ }),
+ ).rejects.toThrow("diagnostic_commit_invalid");
+ await expect(
+ exportDiagnosticZip({
+ dataDir,
+ target: path.join(outputDir, "sha.zip"),
+ productVersion: "1.2.3",
+ coreSha256: "AKIDExampleSecretId1234567890",
+ }),
+ ).rejects.toThrow("diagnostic_sha256_invalid");
+ expect(await readdir(outputDir)).toEqual([]);
+ });
+
+ it("fails closed without publishing a JSON bundle when any line is invalid", async () => {
+ const dataDir = await root();
+ await mkdir(path.join(dataDir, "logs"), { recursive: true });
+ await writeFile(
+ path.join(dataDir, "logs", "diagnostic-core.jsonl"),
+ "not-json\n",
+ { mode: 0o600 },
+ );
+ const outputDir = await root();
+ const target = path.join(outputDir, "bundle.json");
+ await expect(
+ exportDiagnosticBundle({
+ dataDir,
+ target,
+ productVersion: "1.2.3",
+ }),
+ ).rejects.toThrow("diagnostic_export_invalid_lines");
+ expect(await readdir(outputDir)).toEqual([]);
+ });
+
+ it("fails closed when the diagnostic directory cannot be read", async () => {
+ if (process.platform === "win32") return;
+ const dataDir = await root();
+ const logs = path.join(dataDir, "logs");
+ await mkdir(logs, { recursive: true, mode: 0o700 });
+ await writeFile(path.join(logs, "diagnostic-core.jsonl"), "{}\n");
+ const outputDir = await root();
+ const target = path.join(outputDir, "bundle.json");
+ await chmod(logs, 0o000);
+ try {
+ await expect(
+ exportDiagnosticBundle({
+ dataDir,
+ target,
+ productVersion: "1.2.3",
+ }),
+ ).rejects.toMatchObject({ code: "EACCES" });
+ } finally {
+ await chmod(logs, 0o700);
+ }
+ expect(await readdir(outputDir)).toEqual([]);
+ });
+
+ it("never overwrites an existing export target or leaves temporary files", async () => {
+ const dataDir = await root();
+ const outputDir = await root();
+ const target = path.join(outputDir, "bundle.json");
+ await writeFile(target, "existing-artifact", { mode: 0o600 });
+ await expect(
+ exportDiagnosticBundle({
+ dataDir,
+ target,
+ productVersion: "1.2.3",
+ }),
+ ).rejects.toMatchObject({ code: "EEXIST" });
+ expect(await readFile(target, "utf8")).toBe("existing-artifact");
+ expect(await readdir(outputDir)).toEqual(["bundle.json"]);
});
it("preserves microsecond ordering across service log files", async () => {
@@ -311,7 +575,7 @@ describe("diagnostic log", () => {
expect(result.events[1].timestamp).toBe("2026-10-14T00:00:00.123999Z");
});
- it("exports a private seven-day zip with summary, logs, crashes and markers", async () => {
+ it("exports only reprojected events, a strict summary and README", async () => {
const dataDir = await root();
const outputDir = await root();
const crashDir = await root();
@@ -357,39 +621,178 @@ describe("diagnostic log", () => {
markers: [
{
timestamp: "2026-10-05T01:30:00.000Z",
- description: "蓝莓没出来",
+ description:
+ "蓝莓没出来 /Users/private/query.txt ASIA1234567890ABCDEF GOCSPX-clientSecret_A1 Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8 token=ghp_private_token xoxb-123456-abcdef AKIDExampleSecretId1234567890 AIzaSyExampleCredential1234567890 eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signaturevalue",
},
],
summary: {
sources: [{ source_id: "workbuddy", imported_messages: 12 }],
- memory_tasks: { unknown: 1, last_error_codes: { model_failed: 1 } },
+ summary_secret: conversation,
+ background: {
+ status: "exported",
+ runtime_state: "ready",
+ core_state: "healthy",
+ broker_state: "ready",
+ instance_id: "instance-1",
+ secret: conversation,
+ },
},
});
- expect(result.filename).toBe("MemHub.zip");
+ expect(result).toEqual({ filename: "MemHub.zip", file_count: 3 });
const files = unzipSync(await readFile(target));
- expect(Object.keys(files)).toEqual(
- expect.arrayContaining([
- "README.txt",
- "summary.json",
- "logs/diagnostic-core.jsonl",
- "logs/core.log",
- "crashes/MemHub-2026-10-05.ips",
- ]),
- );
- const summary = files["summary.json"];
+ expect(Object.keys(files).sort()).toEqual([
+ "README.txt",
+ "diagnostics/events.jsonl",
+ "summary.json",
+ ]);
+ const eventFile = files["diagnostics/events.jsonl"];
+ const summaryFile = files["summary.json"];
const readme = files["README.txt"];
- if (!summary || !readme) throw new Error("missing support bundle metadata");
- expect(strFromU8(summary)).toContain("model_failed");
- expect(strFromU8(readme)).toContain("蓝莓没出来");
+ if (!eventFile || !summaryFile || !readme)
+ throw new Error("missing support bundle metadata");
+ const exportedEvent = JSON.parse(strFromU8(eventFile).trim());
+ expect(exportedEvent.details).toEqual({ count: 1 });
+ const summary = JSON.parse(strFromU8(summaryFile));
+ expect(summary.background).toEqual({
+ status: "exported",
+ runtime_state: "ready",
+ core_state: "healthy",
+ broker_state: "ready",
+ instance_id: "instance-1",
+ });
+ expect(summary.sources).toBeUndefined();
+ expect(summary.summary_secret).toBeUndefined();
+ expect(summary.problem_markers).toEqual([
+ {
+ timestamp: "2026-10-05T01:30:00.000Z",
+ description: "user_marked_incident",
+ },
+ ]);
+ expect(strFromU8(readme)).toContain("user_marked_incident");
+ expect(strFromU8(readme)).not.toContain("蓝莓没出来");
const all = Object.values(files)
.map((value) => strFromU8(value))
.join("\n");
expect(all).not.toContain(secret);
+ expect(all).not.toContain("ghp_private_token");
+ expect(all).not.toContain("xoxb-123456-abcdef");
+ expect(all).not.toContain("AKIDExampleSecretId1234567890");
+ expect(all).not.toContain("ASIA1234567890ABCDEF");
+ expect(all).not.toContain("GOCSPX-clientSecret_A1");
+ expect(all).not.toContain("Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8");
+ expect(all).not.toContain("AIzaSyExampleCredential1234567890");
+ expect(all).not.toContain("eyJhbGciOiJIUzI1NiJ9");
expect(all).not.toContain(conversation);
expect(all).not.toContain(home);
expect(all).not.toContain("/Users/");
- expect(all).toContain("~/MemHub");
+ });
+
+ it("fails closed without publishing a ZIP when a diagnostic line is invalid", async () => {
+ const dataDir = await root();
+ await mkdir(path.join(dataDir, "logs"), { recursive: true });
+ await writeFile(
+ path.join(dataDir, "logs", "diagnostic-core.jsonl"),
+ "not-json\n",
+ );
+ const outputDir = await root();
+ const target = path.join(outputDir, "MemHub.zip");
+ await expect(
+ exportDiagnosticZip({
+ dataDir,
+ target,
+ productVersion: "3800",
+ }),
+ ).rejects.toThrow("diagnostic_export_invalid_lines");
+ expect(await readdir(outputDir)).toEqual([]);
+ });
+
+ it("stores and exports only the fixed marker description", async () => {
+ const dataDir = await root();
+ const markerFile = path.join(dataDir, "logs", "problem-markers.jsonl");
+ await recordDiagnosticMarker(
+ dataDir,
+ {
+ timestamp: "2026-10-05T01:00:00.000Z",
+ description:
+ "蓝莓 /Users/private/query.txt ASIA1234567890ABCDEF GOCSPX-clientSecret_A1 Aa1Bb2Cc3Dd4Ee5Ff6Gg7Hh8",
+ },
+ "/Users/private",
+ );
+ expect(await readFile(markerFile, "utf8")).toBe(
+ `${JSON.stringify({ timestamp: "2026-10-05T01:00:00.000Z", description: "user_marked_incident" })}\n`,
+ );
+ await writeFile(
+ markerFile,
+ `${await readFile(markerFile, "utf8")}${JSON.stringify({ timestamp: "2026-10-05T01:30:00.000Z", description: "legacy arbitrary text" })}\n`,
+ { mode: 0o600 },
+ );
+
+ const target = path.join(await root(), "markers.zip");
+ await exportDiagnosticZip({
+ dataDir,
+ target,
+ productVersion: "3800",
+ now: new Date("2026-10-05T02:00:00.000Z"),
+ });
+ const files = unzipSync(await readFile(target));
+ const summaryFile = files["summary.json"];
+ if (!summaryFile) throw new Error("missing summary.json");
+ const payload = JSON.stringify(JSON.parse(strFromU8(summaryFile)));
+ expect(payload.match(/user_marked_incident/g)).toHaveLength(2);
+ expect(payload).not.toContain("legacy arbitrary text");
+ expect(payload).not.toContain("蓝莓");
+ expect(payload).not.toContain("/Users/");
+ });
+
+ it("never follows a problem marker symlink for reads or writes", async () => {
+ const dataDir = await root();
+ const logs = path.join(dataDir, "logs");
+ await mkdir(logs, { recursive: true });
+ const outside = path.join(await root(), "outside-marker");
+ await writeFile(outside, "private");
+ await symlink(outside, path.join(logs, "problem-markers.jsonl"));
+
+ await expect(
+ recordDiagnosticMarker(dataDir, {
+ timestamp: "2026-10-05T01:00:00.000Z",
+ description: "anything",
+ }),
+ ).rejects.toThrow("diagnostic_marker_file_unsafe");
+ await expect(
+ exportDiagnosticZip({
+ dataDir,
+ target: path.join(await root(), "markers.zip"),
+ productVersion: "3800",
+ }),
+ ).rejects.toThrow("diagnostic_marker_file_unsafe");
+ expect(await readFile(outside, "utf8")).toBe("private");
+ });
+
+ it("rejects invalid explicit and persisted markers without publishing a ZIP", async () => {
+ const dataDir = await root();
+ const logs = path.join(dataDir, "logs");
+ await mkdir(logs, { recursive: true });
+ const outputDir = await root();
+ const explicitTarget = path.join(outputDir, "explicit.zip");
+ await expect(
+ exportDiagnosticZip({
+ dataDir,
+ target: explicitTarget,
+ productVersion: "3800",
+ markers: [{ timestamp: "invalid", description: "marker" }],
+ }),
+ ).rejects.toThrow("diagnostic_marker_invalid");
+ await writeFile(path.join(logs, "problem-markers.jsonl"), "not-json\n");
+ const persistedTarget = path.join(outputDir, "persisted.zip");
+ await expect(
+ exportDiagnosticZip({
+ dataDir,
+ target: persistedTarget,
+ productVersion: "3800",
+ }),
+ ).rejects.toThrow("diagnostic_marker_invalid");
+ expect(await readdir(outputDir)).toEqual([]);
});
it("an incident retrieves its surrounding session rather than only its marker", async () => {
diff --git a/memhub/packages/runtime-control/src/diagnostic-log.ts b/memhub/packages/runtime-control/src/diagnostic-log.ts
index 8a2979b..53b689b 100644
--- a/memhub/packages/runtime-control/src/diagnostic-log.ts
+++ b/memhub/packages/runtime-control/src/diagnostic-log.ts
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import {
appendFileSync,
chmodSync,
+ constants,
lstatSync,
mkdirSync,
renameSync,
@@ -11,10 +12,11 @@ import {
import {
appendFile,
chmod,
+ link,
lstat,
mkdir,
+ open,
readdir,
- readFile,
rename,
rm,
stat,
@@ -30,7 +32,7 @@ export const DIAGNOSTIC_MAX_FILE_BYTES = 4 * 1024 * 1024;
export const DIAGNOSTIC_MAX_TOTAL_BYTES = 128 * 1024 * 1024;
export const DIAGNOSTIC_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
-export type DiagnosticLevel = "info" | "warn" | "error";
+export type DiagnosticLevel = "debug" | "info" | "warn" | "error";
export type DiagnosticService =
| "runtime"
| "core"
@@ -90,30 +92,49 @@ export interface DiagnosticSink {
const NAME =
/^diagnostic-(runtime|core|host-bridge|desktop|cli)\.jsonl(?:\.\d+)?$/;
+const CODE_FRAME =
+ /^(?:(?:apps|packages|scripts)\/(?:[A-Za-z0-9_-]+\/)*[A-Za-z0-9_.-]+:\d+:\d+|one_memory\/(?:[A-Za-z0-9_-]+\/)*[A-Za-z0-9_.-]+:\d+:[A-Za-z_][A-Za-z0-9_]*|[A-Za-z0-9_][A-Za-z0-9_.-]*\.py:\d+:[A-Za-z_][A-Za-z0-9_]*)$/;
+const THREAD_STACK = /^thread\|(.+)$/;
+const SHORT_CREDENTIAL_TOKEN =
+ /(^|[^A-Za-z0-9])([A-Za-z0-9]{24,47})(?:[_-])?(?=$|[^A-Za-z0-9])/g;
const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
const EVENT_NAME = /^[a-z][a-z0-9]*(?:[._-][a-z0-9]+){0,15}$/;
-const DETAIL_KEYS = new Set([
+const DETAIL_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/;
+const DANGEROUS_DETAIL_KEYS = new Set([
+ "__proto__",
+ "prototype",
+ "constructor",
+]);
+const MAX_DETAIL_KEYS = 32;
+const EXPORT_SECRET_MARKER =
+ /(?:^|[._:-])(?:authorization|credential|password|prompt|query|secret|token)(?:$|[._:-])/i;
+const EXPORTED_STRING_DETAIL_KEYS = new Set([
"action",
- "attempt",
- "count",
"method",
"mode",
"phase",
"platform",
"reason",
- "result_count",
"source_id",
- "spawned",
"status",
"version",
"parent_session_id",
- "invalid_lines",
- "truncated",
- "reader_port",
"runtime_state",
"core_state",
"broker_state",
"instance_id",
+ "operation_kind",
+ "file_kind",
+ "last_probe",
+ "signal",
+]);
+const EXPORTED_NUMBER_DETAIL_KEYS = new Set([
+ "attempt",
+ "count",
+ "status_code",
+ "result_count",
+ "invalid_lines",
+ "reader_port",
"open_file_count",
"memory_bytes",
"thread_count",
@@ -138,18 +159,13 @@ const DETAIL_KEYS = new Set([
"updated_count",
"superseded_count",
"deleted_count",
- "operation_kind",
- "file_kind",
- // Bounded values the Core supervisor reports when a start fails. They are
- // numbers or short enums, not free text, so they are safe to allow explicitly.
"attempts",
"delay_ms",
"budget_ms",
- "last_probe",
"port",
"exit_code",
- "signal",
]);
+const EXPORTED_BOOLEAN_DETAIL_KEYS = new Set(["spawned", "truncated"]);
const EVENT_KEYS = new Set([
"schema_version",
"timestamp",
@@ -283,37 +299,41 @@ export function writeDiagnosticEventSync(
options: DiagnosticLoggerOptions,
input: DiagnosticInput,
): void {
- const event = diagnosticEvent({
- ...buildIdentityFromEnvironment(),
- ...input,
- service: options.service,
- session_id: canonicalUuid(options.sessionId) ?? randomUUID(),
- timestamp:
- input.timestamp ?? (options.now ?? (() => new Date()))().toISOString(),
- });
- const directory = diagnosticDirectory(options.dataDir);
- mkdirSync(directory, { recursive: true, mode: 0o700 });
- const directoryInfo = lstatSync(directory);
- if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink())
- fail("diagnostic_directory_unsafe");
- chmodSync(directory, 0o700);
- const target = path.join(directory, `diagnostic-${options.service}.jsonl`);
- const line = `${JSON.stringify(event)}\n`;
- let current: ReturnType<typeof statSync> | null = null;
try {
- current = statSync(target);
- } catch (error) {
- if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
- }
- if (
- current &&
- current.size + Buffer.byteLength(line) >
- (options.maxFileBytes ?? DIAGNOSTIC_MAX_FILE_BYTES)
- ) {
- rotateSync(target);
+ const event = diagnosticEvent({
+ ...buildIdentityFromEnvironment(),
+ ...input,
+ service: options.service,
+ session_id: canonicalUuid(options.sessionId) ?? randomUUID(),
+ timestamp:
+ input.timestamp ?? (options.now ?? (() => new Date()))().toISOString(),
+ });
+ const directory = diagnosticDirectory(options.dataDir);
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
+ const directoryInfo = lstatSync(directory);
+ if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink())
+ fail("diagnostic_directory_unsafe");
+ chmodSync(directory, 0o700);
+ const target = path.join(directory, `diagnostic-${options.service}.jsonl`);
+ const line = `${JSON.stringify(event)}\n`;
+ let current: ReturnType<typeof statSync> | null = null;
+ try {
+ current = statSync(target);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+ }
+ if (
+ current &&
+ current.size + Buffer.byteLength(line) >
+ (options.maxFileBytes ?? DIAGNOSTIC_MAX_FILE_BYTES)
+ ) {
+ rotateSync(target);
+ }
+ appendFileSync(target, line, { encoding: "utf8", mode: 0o600 });
+ chmodSync(target, 0o600);
+ } catch {
+ // A synchronous fatal-path diagnostic must never replace the business error.
}
- appendFileSync(target, line, { encoding: "utf8", mode: 0o600 });
- chmodSync(target, 0o600);
}
export function diagnosticEvent(
@@ -322,11 +342,17 @@ export function diagnosticEvent(
for (const key of Object.keys(value)) {
if (!EVENT_KEYS.has(key)) fail("diagnostic_field_invalid");
}
+ if (typeof value.timestamp !== "string") fail("diagnostic_timestamp_invalid");
const timestamp = new Date(value.timestamp);
if (Number.isNaN(timestamp.getTime())) fail("diagnostic_timestamp_invalid");
- if (!EVENT_NAME.test(value.component) || !EVENT_NAME.test(value.event))
+ if (
+ typeof value.component !== "string" ||
+ typeof value.event !== "string" ||
+ !EVENT_NAME.test(value.component) ||
+ !EVENT_NAME.test(value.event)
+ )
fail("diagnostic_event_invalid");
- if (!(["info", "warn", "error"] as const).includes(value.level))
+ if (!(["debug", "info", "warn", "error"] as const).includes(value.level))
fail("diagnostic_level_invalid");
if (
!(["runtime", "core", "host-bridge", "desktop", "cli"] as const).includes(
@@ -334,8 +360,10 @@ export function diagnosticEvent(
)
)
fail("diagnostic_service_invalid");
- if (!IDENTIFIER.test(value.outcome)) fail("diagnostic_outcome_invalid");
- if (!canonicalUuid(value.session_id)) fail("diagnostic_session_invalid");
+ if (typeof value.outcome !== "string" || !IDENTIFIER.test(value.outcome))
+ fail("diagnostic_outcome_invalid");
+ if (typeof value.session_id !== "string" || !canonicalUuid(value.session_id))
+ fail("diagnostic_session_invalid");
for (const key of [
"trace_id",
"incident_id",
@@ -349,21 +377,40 @@ export function diagnosticEvent(
"round",
] as const) {
const item = value[key];
- if (item !== undefined && !IDENTIFIER.test(item))
+ if (
+ item !== undefined &&
+ (typeof item !== "string" || !IDENTIFIER.test(item))
+ )
fail("diagnostic_identifier_invalid");
}
+ for (const key of [
+ "state_before",
+ "state_after",
+ "error_code",
+ "error_type",
+ ] as const) {
+ const item = value[key];
+ if (
+ item !== undefined &&
+ (typeof item !== "string" || !IDENTIFIER.test(item))
+ )
+ fail("diagnostic_text_field_invalid");
+ }
if (
value.duration_ms !== undefined &&
(!Number.isSafeInteger(value.duration_ms) || value.duration_ms < 0)
)
fail("diagnostic_duration_invalid");
const details = sanitizeDetails(value.details);
+ const stack =
+ value.safe_stack === undefined ? undefined : safeStack(value.safe_stack);
+ const { details: _details, safe_stack: _safeStack, ...event } = value;
return Object.freeze({
schema_version: DIAGNOSTIC_SCHEMA_VERSION,
- ...value,
+ ...event,
timestamp: timestamp.toISOString(),
...(details ? { details } : {}),
- ...(value.safe_stack ? { safe_stack: safeStack(value.safe_stack) } : {}),
+ ...(stack ? { safe_stack: stack } : {}),
});
}
@@ -388,7 +435,10 @@ export async function readDiagnosticEvents(
let events: DiagnosticEventV1[] = [];
let invalidLines = 0;
for (const file of files) {
- const content = await readFile(file, "utf8");
+ const content = await readRegularFileNoFollow(
+ file,
+ "diagnostic_file_unsafe",
+ );
for (const line of content.split("\n")) {
if (!line) continue;
try {
@@ -441,16 +491,18 @@ export async function exportDiagnosticBundle(options: {
status?: Record<string, string | number | boolean | null>;
}): Promise<{ event_count: number; invalid_lines: number }> {
const result = await readDiagnosticEvents(options.dataDir, { tail: 5_000 });
+ if (result.invalid_lines > 0) fail("diagnostic_export_invalid_lines");
+ const events = result.events.map(projectDiagnosticEventForSafeOutput);
const payload = {
schema_version: 1,
exported_at: new Date().toISOString(),
- product_version: cleanText(options.productVersion, 128),
+ product_version: exportProductVersion(options.productVersion),
platform: process.platform,
arch: process.arch,
- status: sanitizeDetails(options.status),
- event_count: result.events.length,
- invalid_lines: result.invalid_lines,
- events: result.events,
+ status: projectExportDetails(options.status),
+ event_count: events.length,
+ invalid_lines: 0,
+ events,
};
const directory = path.dirname(options.target);
await mkdir(directory, { recursive: true });
@@ -465,8 +517,7 @@ export async function exportDiagnosticBundle(options: {
flag: "wx",
});
await chmod(temporary, 0o600);
- await rename(temporary, options.target);
- await chmod(options.target, 0o600);
+ await publishTemporaryFile(temporary, options.target);
} catch (error) {
await rm(temporary, { force: true }).catch(() => undefined);
throw error;
@@ -477,72 +528,68 @@ export async function exportDiagnosticBundle(options: {
};
}
-const SUPPORT_LOG =
- /^(?:diagnostic-(?:runtime|core|host-bridge|desktop|cli)\.jsonl|core\.log|service-error\.log|service\.log|recall-inject\.jsonl)(?:\.\d+)?$/;
-const CRASH_REPORT = /^MemHub.*\.(?:ips|crash)$/i;
-const PRIVATE_FIELD =
- /(?:authorization|api[_-]?key|token|secret|credential|password)|^(?:prompt|query|content|messages?|memory|memory_text|raw_text)$/i;
-
export interface DiagnosticMarker {
timestamp: string;
description: string;
}
const MARKER_FILE = "problem-markers.jsonl";
+const MARKER_DESCRIPTION = "user_marked_incident";
export async function recordDiagnosticMarker(
dataDir: string,
marker: DiagnosticMarker,
home = os.homedir(),
): Promise<void> {
- const timestamp = new Date(marker.timestamp);
- if (Number.isNaN(timestamp.getTime())) fail("diagnostic_timestamp_invalid");
+ const safe = validateDiagnosticMarker(marker, home);
const directory = await ensureDiagnosticDirectory(dataDir);
- const safe = {
- timestamp: timestamp.toISOString(),
- description: redactString(cleanText(marker.description, 500), home),
- };
- await appendFile(
- path.join(directory, MARKER_FILE),
- `${JSON.stringify(safe)}\n`,
- {
- encoding: "utf8",
- mode: 0o600,
- },
+ const target = path.join(directory, MARKER_FILE);
+ await refuseUnsafeExistingFile(target, "diagnostic_marker_file_unsafe");
+ const handle = await open(
+ target,
+ constants.O_WRONLY |
+ constants.O_CREAT |
+ constants.O_APPEND |
+ (constants.O_NOFOLLOW ?? 0),
+ 0o600,
);
+ try {
+ const info = await handle.stat();
+ if (!info.isFile()) fail("diagnostic_marker_file_unsafe");
+ await handle.appendFile(`${JSON.stringify(safe)}\n`, "utf8");
+ await handle.chmod(0o600);
+ } finally {
+ await handle.close();
+ }
}
async function readDiagnosticMarkers(
dataDir: string,
home: string,
): Promise<DiagnosticMarker[]> {
- const raw = await readFile(
- path.join(diagnosticDirectory(dataDir), MARKER_FILE),
- "utf8",
- ).catch(() => "");
- return raw
- .split("\n")
- .flatMap((line) => {
- if (!line) return [];
- try {
- const value = JSON.parse(line) as Record<string, unknown>;
- if (
- typeof value.timestamp !== "string" ||
- Number.isNaN(new Date(value.timestamp).getTime()) ||
- typeof value.description !== "string"
- )
- return [];
- return [
- {
- timestamp: new Date(value.timestamp).toISOString(),
- description: redactString(cleanText(value.description, 500), home),
- },
- ];
- } catch {
- return [];
- }
- })
- .slice(-50);
+ const target = path.join(diagnosticDirectory(dataDir), MARKER_FILE);
+ let raw: string;
+ try {
+ raw = await readRegularFileNoFollow(
+ target,
+ "diagnostic_marker_file_unsafe",
+ );
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+ throw error;
+ }
+ const markers: DiagnosticMarker[] = [];
+ for (const line of raw.split("\n")) {
+ if (!line) continue;
+ let value: unknown;
+ try {
+ value = JSON.parse(line);
+ } catch {
+ fail("diagnostic_marker_invalid");
+ }
+ markers.push(validateDiagnosticMarker(value, home));
+ }
+ return markers.slice(-50);
}
export function diagnosticZipFilename(build: string, now = new Date()): string {
@@ -566,97 +613,62 @@ export async function exportDiagnosticZip(options: {
}): Promise<{ filename: string; file_count: number }> {
const now = options.now ?? new Date();
const home = options.home ?? os.homedir();
- const archive: Record<string, Uint8Array> = {};
- const logsDir = diagnosticDirectory(options.dataDir);
- const cutoff = now.getTime() - 7 * 24 * 60 * 60 * 1000;
- const logEntries = await readdir(logsDir, { withFileTypes: true }).catch(
- () => [],
- );
- for (const entry of logEntries) {
- if (
- !entry.isFile() ||
- entry.isSymbolicLink() ||
- !SUPPORT_LOG.test(entry.name)
- )
- continue;
- const source = path.join(logsDir, entry.name);
- const info = await lstat(source);
- if (!info.isFile() || info.isSymbolicLink() || info.mtimeMs < cutoff)
- continue;
- archive[`logs/${entry.name}`] = strToU8(
- sanitizeDiagnosticText(await readFile(source, "utf8"), home),
- );
- }
- const crashCutoff = now.getTime() - 24 * 60 * 60 * 1000;
- if (options.crashReportsDir) {
- const crashEntries = await readdir(options.crashReportsDir, {
- withFileTypes: true,
- }).catch(() => []);
- for (const entry of crashEntries) {
- if (
- !entry.isFile() ||
- entry.isSymbolicLink() ||
- !CRASH_REPORT.test(entry.name)
- )
- continue;
- const source = path.join(options.crashReportsDir, entry.name);
- const info = await lstat(source);
- if (!info.isFile() || info.isSymbolicLink() || info.mtimeMs < crashCutoff)
- continue;
- archive[`crashes/${entry.name}`] = strToU8(
- sanitizeDiagnosticText(await readFile(source, "utf8"), home),
- );
- }
- }
+ const cutoff = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
+ const result = await readDiagnosticEvents(options.dataDir, {
+ since: cutoff,
+ tail: 5_000,
+ });
+ if (result.invalid_lines > 0) fail("diagnostic_export_invalid_lines");
+ const events = result.events.map(projectDiagnosticEventForSafeOutput);
+ const markers = options.markers
+ ? options.markers
+ .map((marker) => validateDiagnosticMarker(marker, home))
+ .slice(-50)
+ : await readDiagnosticMarkers(options.dataDir, home);
const disk = await statfs(options.dataDir).catch(() => null);
- const markers = (
- options.markers ?? (await readDiagnosticMarkers(options.dataDir, home))
- ).map((marker) => ({
- timestamp: new Date(marker.timestamp).toISOString(),
- description: cleanText(marker.description, 500),
- }));
- const summary = sanitizeBundleValue(
- {
- schema_version: 1,
- exported_at: now.toISOString(),
- build: cleanText(options.productVersion, 128),
- commit: options.commit ? cleanText(options.commit, 64) : null,
- core_sha256: options.coreSha256
- ? cleanText(options.coreSha256, 128)
- : null,
- system: {
- macos_version: process.platform === "darwin" ? os.release() : null,
- platform: process.platform,
- arch: process.arch,
- chip: os.cpus()[0]?.model ?? "unknown",
- memory_bytes: os.totalmem(),
- free_disk_bytes: disk ? disk.bavail * disk.bsize : null,
- },
- problem_markers: markers,
- ...(options.summary ?? {}),
+ const background = projectSummaryBackground(options.summary);
+ const summary = {
+ schema_version: 1,
+ exported_at: now.toISOString(),
+ build: exportProductVersion(options.productVersion),
+ commit: options.commit ? exportCommit(options.commit) : null,
+ core_sha256: options.coreSha256 ? exportSha256(options.coreSha256) : null,
+ system: {
+ macos_version: process.platform === "darwin" ? os.release() : null,
+ platform: process.platform,
+ arch: process.arch,
+ chip: os.cpus()[0]?.model ?? "unknown",
+ memory_bytes: os.totalmem(),
+ free_disk_bytes: disk ? disk.bavail * disk.bsize : null,
},
- home,
- );
- archive["summary.json"] = strToU8(`${JSON.stringify(summary, null, 2)}\n`);
- archive["README.txt"] = strToU8(
- [
- "MemHub diagnostics",
- "",
- "包含:最近 7 天诊断日志、运行日志、最近 24 小时 MemHub 崩溃报告,以及不含正文的状态摘要。",
- "不包含:对话原文、记忆内容、API Key、token 或宿主凭证。",
- "本机主目录已替换为 ~。",
- ...(markers.length
- ? [
- "",
- "问题时间点:",
- ...markers.map(
- (marker) => `${marker.timestamp} ${marker.description}`,
- ),
- ]
- : []),
- "",
- ].join("\n"),
- );
+ problem_markers: markers,
+ ...(background ? { background } : {}),
+ };
+ const eventLines = events.map((event) => JSON.stringify(event)).join("\n");
+ const archive: Record<string, Uint8Array> = {
+ "diagnostics/events.jsonl": strToU8(
+ eventLines.length > 0 ? `${eventLines}\n` : "",
+ ),
+ "summary.json": strToU8(`${JSON.stringify(summary, null, 2)}\n`),
+ "README.txt": strToU8(
+ [
+ "MemHub diagnostics",
+ "",
+ "包含:最近 7 天通过安全白名单重新投影的诊断事件,以及不含正文的状态摘要。",
+ "不包含:原始日志、崩溃报告、对话原文、记忆内容、API Key、token 或宿主凭证。",
+ ...(markers.length
+ ? [
+ "",
+ "问题时间点:",
+ ...markers.map(
+ (marker) => `${marker.timestamp} ${marker.description}`,
+ ),
+ ]
+ : []),
+ "",
+ ].join("\n"),
+ ),
+ };
const directory = path.dirname(options.target);
await mkdir(directory, { recursive: true });
const temporary = path.join(directory, `.memhub-logs-${randomUUID()}.tmp`);
@@ -665,8 +677,8 @@ export async function exportDiagnosticZip(options: {
mode: 0o600,
flag: "wx",
});
- await rename(temporary, options.target);
- await chmod(options.target, 0o600);
+ await chmod(temporary, 0o600);
+ await publishTemporaryFile(temporary, options.target);
} catch (error) {
await rm(temporary, { force: true }).catch(() => undefined);
throw error;
@@ -677,61 +689,214 @@ export async function exportDiagnosticZip(options: {
};
}
-function sanitizeDiagnosticText(value: string, home: string): string {
- return value
- .split("\n")
- .map((line) => {
- if (!line) return "";
- try {
- return JSON.stringify(sanitizeBundleValue(JSON.parse(line), home));
- } catch {
- return redactString(line, home)
- .replace(
- /\b(?:authorization|api[_-]?key|token|secret|credential|password)\s*[:=]\s*\S+/gi,
- (match) => match.replace(/[:=].*$/, "=[redacted]"),
- )
- .replace(
- /\b(?:content|prompt|query|message|memory_text|raw_text)\s*[:=].*$/gi,
- (match) => match.replace(/[:=].*$/, "=[redacted]"),
- );
- }
- })
- .join("\n");
-}
-
-function sanitizeBundleValue(value: unknown, home: string, key = ""): unknown {
- if (PRIVATE_FIELD.test(key)) return "[redacted]";
- if (typeof value === "string") return redactString(value, home);
- if (Array.isArray(value))
- return value.map((item) => sanitizeBundleValue(item, home));
- if (value && typeof value === "object")
- return Object.fromEntries(
- Object.entries(value).map(([childKey, child]) => [
- childKey,
- sanitizeBundleValue(child, home, childKey),
- ]),
- );
- return value;
+async function publishTemporaryFile(
+ temporary: string,
+ target: string,
+): Promise<void> {
+ // A hard link publishes the already-permissioned inode atomically and refuses
+ // to overwrite an existing target. That avoids both stale-file confusion and
+ // the rename-then-chmod window where a later failure leaves a false success.
+ await link(temporary, target);
+ await rm(temporary, { force: true }).catch(() => undefined);
+}
+
+export function projectDiagnosticEventForSafeOutput(
+ event: DiagnosticEventV1,
+): DiagnosticEventV1 {
+ const details = projectExportDetails(event.details);
+ const stack = projectSafeStack(event.safe_stack);
+ const optional: Partial<DiagnosticEventV1> = {};
+ for (const key of [
+ "trace_id",
+ "incident_id",
+ "instance_id",
+ "operation_id",
+ "request_id",
+ "job_id",
+ "task_id",
+ "build_id",
+ "package_id",
+ "round",
+ "state_before",
+ "state_after",
+ "error_code",
+ "error_type",
+ ] as const) {
+ const value = event[key];
+ if (value !== undefined && isExportIdentifier(value)) optional[key] = value;
+ }
+ return {
+ schema_version: DIAGNOSTIC_SCHEMA_VERSION,
+ timestamp: event.timestamp,
+ level: event.level,
+ service: event.service,
+ component: isExportIdentifier(event.component)
+ ? event.component
+ : "diagnostic.redacted",
+ event: isExportIdentifier(event.event)
+ ? event.event
+ : "diagnostic.redacted",
+ outcome: isExportIdentifier(event.outcome) ? event.outcome : "redacted",
+ session_id: event.session_id,
+ ...optional,
+ ...(event.duration_ms === undefined ||
+ !Number.isSafeInteger(event.duration_ms)
+ ? {}
+ : { duration_ms: event.duration_ms }),
+ ...(stack === undefined ? {} : { safe_stack: stack }),
+ ...(details === undefined ? {} : { details }),
+ };
+}
+
+function projectExportDetails(
+ value: Record<string, unknown> | undefined,
+): Record<string, string | number | boolean | null> | undefined {
+ if (!value || typeof value !== "object" || Array.isArray(value))
+ return undefined;
+ const result: Record<string, string | number | boolean | null> = {};
+ for (const [key, item] of Object.entries(value)) {
+ if (
+ EXPORTED_STRING_DETAIL_KEYS.has(key) &&
+ typeof item === "string" &&
+ isExportIdentifier(item)
+ ) {
+ result[key] = item;
+ } else if (
+ EXPORTED_NUMBER_DETAIL_KEYS.has(key) &&
+ typeof item === "number" &&
+ isJsonSafeNumber(item)
+ ) {
+ result[key] = item;
+ } else if (
+ EXPORTED_BOOLEAN_DETAIL_KEYS.has(key) &&
+ typeof item === "boolean"
+ ) {
+ result[key] = item;
+ }
+ }
+ return Object.keys(result).length > 0 ? result : undefined;
+}
+
+function validateDiagnosticMarker(
+ value: unknown,
+ _home: string,
+): DiagnosticMarker {
+ if (!value || typeof value !== "object" || Array.isArray(value))
+ fail("diagnostic_marker_invalid");
+ const marker = value as Record<string, unknown>;
+ const keys = Object.keys(marker);
+ if (
+ keys.length !== 2 ||
+ !keys.includes("timestamp") ||
+ !keys.includes("description") ||
+ typeof marker.timestamp !== "string" ||
+ typeof marker.description !== "string"
+ )
+ fail("diagnostic_marker_invalid");
+ const timestamp = new Date(marker.timestamp);
+ if (Number.isNaN(timestamp.getTime())) fail("diagnostic_marker_invalid");
+ return {
+ timestamp: timestamp.toISOString(),
+ description: redactString(MARKER_DESCRIPTION, _home),
+ };
+}
+
+function projectSummaryBackground(
+ summary: Record<string, unknown> | undefined,
+): Record<string, string> | undefined {
+ if (!summary || typeof summary !== "object" || Array.isArray(summary))
+ return undefined;
+ const candidate = summary.background;
+ if (!candidate || typeof candidate !== "object" || Array.isArray(candidate))
+ return undefined;
+ const source = candidate as Record<string, unknown>;
+ const background: Record<string, string> = {};
+ for (const key of [
+ "status",
+ "runtime_state",
+ "core_state",
+ "broker_state",
+ "instance_id",
+ ]) {
+ const value = source[key];
+ if (typeof value === "string" && isExportIdentifier(value))
+ background[key] = value;
+ }
+ return Object.keys(background).length > 0 ? background : undefined;
}
function redactString(value: string, home: string): string {
- const withoutHome = (home ? value.split(home).join("~") : value).replace(
- /\/Users\/[^/\s"']+/g,
- "~",
- );
+ const withoutHome = (home ? value.split(home).join("~") : value)
+ .replace(/\/Users\/[^/\s"']+/g, "~")
+ .replace(/\b[A-Za-z]:\\Users\\[^\\\s"']+/g, "~");
return withoutHome
.replace(/\bBearer\s+[^\s"']+/gi, "Bearer [redacted]")
- .replace(/\b(?:sk|omhg)_[A-Za-z0-9._-]+\b/g, "[redacted]")
- .replace(/\bsk-[A-Za-z0-9._-]+\b/g, "[redacted]");
+ .replace(/\b(?:sk|omhg|ghp|github_pat)_[A-Za-z0-9._-]+\b/gi, "[redacted]")
+ .replace(/\bsk-[A-Za-z0-9._-]+\b/gi, "[redacted]")
+ .replace(/\bxox[baprs]-[A-Za-z0-9-]+\b/gi, "[redacted]")
+ .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, "[redacted]")
+ .replace(/\bAKID[A-Za-z0-9]{12,}\b/g, "[redacted]")
+ .replace(/\bAIza[A-Za-z0-9_-]{20,}\b/g, "[redacted]")
+ .replace(/\bGOCSPX-[A-Za-z0-9_-]+\b/g, "[redacted]")
+ .replace(SHORT_CREDENTIAL_TOKEN, (match, prefix: string, token: string) =>
+ isConservativeShortCredential(token) ? `${prefix}[redacted]` : match,
+ )
+ .replace(
+ /\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
+ "[redacted]",
+ )
+ .replace(
+ /\b(?=[A-Za-z0-9._-]{48,}\b)(?=[A-Za-z0-9._-]*[A-Za-z])(?=[A-Za-z0-9._-]*\d)[A-Za-z0-9._-]+\b/g,
+ "[redacted]",
+ )
+ .replace(
+ /\b(api[_-]?key|authorization|credential|password|secret|token)\s*[:=]\s*[^\s,;"']+/gi,
+ "$1=[redacted]",
+ );
+}
+
+async function refuseUnsafeExistingFile(
+ target: string,
+ code: string,
+): Promise<void> {
+ try {
+ const info = await lstat(target);
+ if (!info.isFile() || info.isSymbolicLink()) fail(code);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+ }
+}
+
+async function readRegularFileNoFollow(
+ target: string,
+ code: string,
+): Promise<string> {
+ await refuseUnsafeExistingFile(target, code);
+ const handle = await open(
+ target,
+ constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
+ );
+ try {
+ const info = await handle.stat();
+ if (!info.isFile()) fail(code);
+ return await handle.readFile("utf8");
+ } finally {
+ await handle.close();
+ }
}
export async function diagnosticLogFiles(dataDir: string): Promise<string[]> {
const directory = diagnosticDirectory(dataDir);
- const directoryInfo = await lstat(directory).catch(() => null);
- if (directoryInfo?.isSymbolicLink()) return [];
- const entries = await readdir(directory, { withFileTypes: true }).catch(
- () => [],
- );
+ let directoryInfo: Awaited<ReturnType<typeof lstat>>;
+ try {
+ directoryInfo = await lstat(directory);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+ throw error;
+ }
+ if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink())
+ fail("diagnostic_directory_unsafe");
+ const entries = await readdir(directory, { withFileTypes: true });
const files: Array<{ path: string; mtime: number }> = [];
for (const entry of entries) {
if (!entry.isFile() || entry.isSymbolicLink() || !NAME.test(entry.name))
@@ -858,7 +1023,7 @@ function compareDiagnosticTimestamps(left: string, right: string): number {
const milliseconds = Date.parse(value);
if (!Number.isFinite(milliseconds)) fail("diagnostic_timestamp_invalid");
const fraction = /\.(\d+)(?:Z|[+-]\d{2}:\d{2})$/.exec(value)?.[1] ?? "";
- const nanoseconds = BigInt((fraction + "000000000").slice(0, 9));
+ const nanoseconds = BigInt(`${fraction}000000000`.slice(0, 9));
return (
BigInt(Math.floor(milliseconds / 1000)) * 1_000_000_000n + nanoseconds
);
@@ -868,15 +1033,80 @@ function compareDiagnosticTimestamps(left: string, right: string): number {
return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0;
}
+function containsCredentialMaterial(value: string): boolean {
+ return (
+ /(?:^|[^A-Za-z0-9])(?:sk-|sk_|omhg_|ghp_|github_pat_)[A-Za-z0-9._-]+/i.test(
+ value,
+ ) ||
+ /(?:^|[^A-Za-z0-9])xox[baprs]-[A-Za-z0-9-]+/i.test(value) ||
+ /(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}(?![A-Za-z0-9])/.test(value) ||
+ /(?:^|[^A-Za-z0-9])AKID[A-Za-z0-9]{12,}/i.test(value) ||
+ /(?:^|[^A-Za-z0-9])AIza[A-Za-z0-9_-]{20,}/.test(value) ||
+ /(?:^|[^A-Za-z0-9])GOCSPX-[A-Za-z0-9_-]+/.test(value) ||
+ hasConservativeShortCredential(value) ||
+ /\b[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/.test(
+ value,
+ ) ||
+ /\b(?=[A-Za-z0-9._-]{48,}\b)(?=[A-Za-z0-9._-]*[A-Za-z])(?=[A-Za-z0-9._-]*\d)[A-Za-z0-9._-]+\b/.test(
+ value,
+ )
+ );
+}
+
+function hasConservativeShortCredential(value: string): boolean {
+ for (const match of value.matchAll(SHORT_CREDENTIAL_TOKEN)) {
+ if (match[2] && isConservativeShortCredential(match[2])) return true;
+ }
+ return false;
+}
+
+function isConservativeShortCredential(value: string): boolean {
+ return (
+ /[A-Z]/.test(value) &&
+ /[a-z]/.test(value) &&
+ /\d/.test(value) &&
+ !/^[A-Fa-f0-9]+$/.test(value)
+ );
+}
+
+function looksLikeCredential(value: string): boolean {
+ return containsCredentialMaterial(value);
+}
+
+function isExportIdentifier(value: string): boolean {
+ return (
+ IDENTIFIER.test(value) &&
+ !EXPORT_SECRET_MARKER.test(value) &&
+ !looksLikeCredential(value)
+ );
+}
+
+function isJsonSafeNumber(value: number): boolean {
+ return (
+ Number.isFinite(value) &&
+ (!Number.isInteger(value) || Number.isSafeInteger(value))
+ );
+}
+
function sanitizeDetails(
value: Record<string, unknown> | undefined,
): Record<string, string | number | boolean | null> | undefined {
- if (!value) return undefined;
+ if (value === undefined) return undefined;
+ if (!value || typeof value !== "object" || Array.isArray(value))
+ fail("diagnostic_details_invalid");
+ const prototype = Object.getPrototypeOf(value);
+ if (prototype !== Object.prototype && prototype !== null)
+ fail("diagnostic_detail_key_invalid");
+ const keys = Reflect.ownKeys(value);
+ if (keys.length > MAX_DETAIL_KEYS) fail("diagnostic_detail_count_invalid");
+ if (keys.some((key) => typeof key !== "string"))
+ fail("diagnostic_detail_key_invalid");
const result: Record<string, string | number | boolean | null> = {};
for (const [key, item] of Object.entries(value)) {
- if (!DETAIL_KEYS.has(key)) fail("diagnostic_detail_key_invalid");
+ if (!DETAIL_KEY.test(key) || DANGEROUS_DETAIL_KEYS.has(key))
+ fail("diagnostic_detail_key_invalid");
if (item === null || typeof item === "boolean") result[key] = item;
- else if (typeof item === "number" && Number.isFinite(item))
+ else if (typeof item === "number" && isJsonSafeNumber(item))
result[key] = item;
else if (typeof item === "string") result[key] = cleanText(item, 256);
else fail("diagnostic_detail_value_invalid");
@@ -884,17 +1114,57 @@ function sanitizeDetails(
return Object.keys(result).length ? result : undefined;
}
+function projectSafeStack(value: string[] | undefined): string[] | undefined {
+ if (!Array.isArray(value)) return undefined;
+ const result = value
+ .filter((line) => typeof line === "string" && isSafeStackLine(line))
+ .slice(0, 16);
+ return result.length > 0 ? result : undefined;
+}
+
function safeStack(value: string[]): string[] {
if (!Array.isArray(value) || value.length > 16)
fail("diagnostic_stack_invalid");
return value.map((line) => {
- const cleaned = cleanText(line, 256);
- if (path.isAbsolute(cleaned) || cleaned.includes("file://"))
+ if (typeof line !== "string" || !isSafeStackLine(line))
fail("diagnostic_stack_invalid");
- return cleaned;
+ return line;
});
}
+function isSafeStackLine(value: string): boolean {
+ if (
+ value.length < 1 ||
+ value.length > 256 ||
+ /[\0\r\n]/.test(value) ||
+ containsCredentialMaterial(value)
+ )
+ return false;
+ if (CODE_FRAME.test(value)) return true;
+ const aggregate = THREAD_STACK.exec(value)?.[1];
+ return (
+ aggregate?.split(" > ").every((frame) => CODE_FRAME.test(frame)) === true
+ );
+}
+
+function exportProductVersion(value: string): string {
+ const cleaned = cleanText(value, 128);
+ if (!isExportIdentifier(cleaned)) fail("diagnostic_product_version_invalid");
+ return cleaned;
+}
+
+function exportCommit(value: string): string {
+ const cleaned = cleanText(value, 64);
+ if (!/^[0-9a-f]{7,64}$/i.test(cleaned)) fail("diagnostic_commit_invalid");
+ return cleaned;
+}
+
+function exportSha256(value: string): string {
+ const cleaned = cleanText(value, 64);
+ if (!/^[0-9a-f]{64}$/i.test(cleaned)) fail("diagnostic_sha256_invalid");
+ return cleaned;
+}
+
function cleanText(value: string, maximum: number): string {
if (!value || value.length > maximum || /[\0\r\n]/.test(value))
fail("diagnostic_text_invalid");
diff --git a/memhub/packages/runtime-control/src/index.ts b/memhub/packages/runtime-control/src/index.ts
index 4ece6cb..cd1331d 100644
--- a/memhub/packages/runtime-control/src/index.ts
+++ b/memhub/packages/runtime-control/src/index.ts
@@ -59,6 +59,7 @@ export {
diagnosticZipFilename,
exportDiagnosticBundle,
exportDiagnosticZip,
+ projectDiagnosticEventForSafeOutput,
pruneDiagnosticLogs,
readDiagnosticEvents,
recordDiagnosticMarker,