Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Layered Intelligence — source gathering I/O (#2842 split of layeredIntelligence.js).
*
* Reads each configured source (planned work, files, HTTP, shell, LI task metrics)
* with injectable deps so tests drive them without a live filesystem, network or
* forge CLI. Forge CLI calls go through lib/runCliBounded.js.
*/
import { join, resolve, relative, isAbsolute } from 'path';
import { existsSync } from 'fs';
import { realpath } from 'fs/promises';
import { tryReadFile, readJSONFile, PATHS } from '../../lib/fileUtils.js';
import { bufferedSpawn } from '../../lib/bufferedSpawn.js';
import { fetchPublicText } from '../../lib/safeUrlFetch.js';
import { validateUnattendedCommand } from '../../lib/commandSecurity.js';
import { getSettings } from '../settings.js';
import { computeWindowedStats } from '../taskLearning/store.js';
import { computeChurn, summarizeRecentRuns, SHORT_LIVED_MS } from '../agentChurn.js';
import {
PLANNED_WORK_LABEL, PLANNED_WORK_NONE, PLANNED_WORK_UNAVAILABLE_PREFIX,
PLANNED_WORK_MAX_ITEMS, PLANNED_WORK_MAX_CHARS, LI_TASK_TYPE,
} from './constants.js';
import { computeScopeAwareness } from './awareness.js';
import { listForgeIssues, extractIssuePriority } from './forgeFiler.js';
import { listJiraIssues, plannedWorkJql } from './jiraFiler.js';
// ---------------------------------------------------------------------------
// I/O layer — gather + filers. Injectable deps keep these testable.
// ---------------------------------------------------------------------------
/**
* Why a planned-work read came back empty-handed. Rendered INSTEAD of a listing so
* a failed tracker read can never be mistaken for "this app has nothing planned" —
* the two are opposite instructions to the reasoner (be conservative vs. the field
* is wide open), and collapsing them is exactly the failure the sentinel rule in
* AGENTS.md exists to prevent.
*/
export function plannedWorkUnavailable(why) {
return `${PLANNED_WORK_UNAVAILABLE_PREFIX} (${why}). Do NOT treat this as "nothing is planned" — this app may well have a committed backlog that simply could not be listed this run. Be conservative: prefer proposal: null over filing work that might already be in scope.`;
}
/**
* Whether a gathered plannedWork string is an actual LISTING of committed work —
* i.e. something the reasoner can be told to go read — as opposed to one of the
* two sentinels ("nothing is planned" / "could not be read"). Both sentinels are
* meaningful in the prompt and still render, but neither is a backlog: telling
* the reasoner its proposals "may be overlapping with committed work — review the
* plannedWork source above" directly beneath a block stating no planned work
* exists is a contradiction that just biases it toward filing nothing.
*/
export function hasPlannedWorkListing(plannedWork) {
if (typeof plannedWork !== 'string') return false;
const text = plannedWork.trim();
if (!text || text === PLANNED_WORK_NONE) return false;
return !text.startsWith(PLANNED_WORK_UNAVAILABLE_PREFIX);
}
/**
* Render a planned-work item list into the prompt block. Pure.
*
* Reports the count of everything it was HANDED even when the rendered list is
* truncated to `maxItems`, so the reasoner knows it is seeing the top of a larger
* backlog rather than all of it — 15 shown out of 100 must not read as "there are
* only 15". Note the caller's own read is capped too (the tracker lists cap at
* 100), so on a very large backlog this count is itself a floor, not a census.
*/
export function formatPlannedWork(items, { maxItems = PLANNED_WORK_MAX_ITEMS, maxChars = PLANNED_WORK_MAX_CHARS } = {}) {
const list = (Array.isArray(items) ? items : []).filter(i => i && typeof i === 'object' && (i.title || i.number != null));
if (list.length === 0) return PLANNED_WORK_NONE;
const top = list.slice(0, maxItems);
const lines = top.map(i => {
const ref = i.number != null ? `#${i.number} ` : '';
const meta = [
i.priority ? `priority: ${i.priority}` : null,
Array.isArray(i.labels) && i.labels.length ? `labels: ${i.labels.join(', ')}` : null
].filter(Boolean).join('; ');
return `- ${ref}${(i.title || '').trim()}${meta ? ` (${meta})` : ''}`;
});
const header = list.length > top.length
? `${list.length} items of actively-planned work the user has already committed to (showing the top ${top.length}):`
: `${list.length} item(s) of actively-planned work the user has already committed to:`;
return [header, ...lines].join('\n').slice(0, maxChars);
}
/**
* Extract a PLAN.md's UNCHECKED (`- [ ]`) items — the plan tracker's equivalent of
* a `plan`-labeled issue. A `- [x]` item is finished work, not committed-and-
* pending, so it is excluded: proposing something already done is a different
* (and already-handled) problem than proposing something already scheduled.
*/
export function extractPlannedPlanItems(planContent) {
if (typeof planContent !== 'string') return [];
const items = [];
const re = /^[ \t]*[-*][ \t]+\[ \][ \t]*(\S.*)$/gm;
let m;
while ((m = re.exec(planContent))) {
const title = m[1].replace(/\s+/g, ' ').trim();
if (title) items.push({ number: null, title: title.slice(0, 200), labels: [], priority: null });
}
return items;
}
/**
* Gather the app's actively-planned work — the backlog the user has ALREADY
* committed to — so the reasoner can cross-reference a proposal against it before
* filing (#2698). A deterministic tracker read: files + `gh`/`glab`/Jira REST, and
* NO LLM call (the no-cold-bootstrap rule).
*
* Returns a prompt-ready string, or `null` when the source does not apply at all
* (an unresolvable tracker: no forge CLI, no Jira coords, no repo path) — three
* distinct states, never collapsed:
* - `null` → nothing to say; buildPrompt omits the block
* - `plannedWorkUnavailable()` → the read FAILED; be conservative
* - a listing / PLANNED_WORK_NONE → the read SUCCEEDED and is trustworthy
* (a plan-tracked app with no PLAN.md at all is a real PLANNED_WORK_NONE)
*
* Deps are injectable so tests drive every branch without a live tracker.
*/
export async function gatherPlannedWork({
filer,
forgeCli,
cwd,
jira,
listForge = listForgeIssues,
listJira = listJiraIssues,
readFileFn = tryReadFile
} = {}) {
if (filer === 'forge' && forgeCli && cwd) {
const { ok, issues } = await listForge({ cli: forgeCli, cwd, label: PLANNED_WORK_LABEL, state: 'open' });
if (!ok) return plannedWorkUnavailable(`the ${forgeCli} issue list failed`);
// gh honors --state open, but glab's label list can still surface a closed
// issue depending on version — re-filter so a done item never reads as pending.
const open = issues.filter(i => i.state === 'open');
return formatPlannedWork(open.map(i => ({
number: i.number,
title: i.title,
labels: i.labels || [],
priority: extractIssuePriority(i.labels)
})));
}
if (filer === 'jira' && jira?.instanceId && jira?.projectKey) {
const { ok, issues } = await listJira({
instanceId: jira.instanceId,
projectKey: jira.projectKey,
jql: plannedWorkJql(jira.projectKey),
// Jira's priority field is not in searchIssues' default `fields` set — ask
// for it explicitly, or every item would report a null priority.
searchOptions: { fields: 'summary,status,labels,updated,description,resolutiondate,priority' }
});
if (!ok) return plannedWorkUnavailable('the Jira search failed');
const open = issues.filter(i => i.state === 'open');
return formatPlannedWork(open.map(i => ({
number: i.number,
title: i.title,
labels: i.labels || [],
// Prefer Jira's real priority field; fall back to a priority-ish label.
priority: i.priority || extractIssuePriority(i.labels)
})));
}
if (filer === 'plan' && cwd) {
const planPath = join(cwd, 'PLAN.md');
// No PLAN.md at all is a real "nothing is planned" for a plan-tracked app —
// distinguish it from a PLAN.md that EXISTS but could not be read (a genuine
// failure), which tryReadFile's null would otherwise conflate.
if (!existsSync(planPath)) return PLANNED_WORK_NONE;
const content = await readFileFn(planPath);
if (typeof content !== 'string') return plannedWorkUnavailable('PLAN.md exists but could not be read');
return formatPlannedWork(extractPlannedPlanItems(content));
}
return null;
}
// Prompt-budget cap on the rendered cosMetrics document, unchanged from when the
// per-task-type map was the whole payload.
export const COS_METRICS_MAX_CHARS = 4000;
// The two `cosMetrics` keys that are NOT task types (#3085). Named so the reserved-key
// contract is stated once rather than implied by assignment order below.
const COS_METRICS_DELIVERY_KEYS = ['delivery', 'deliveryByScope'];
/**
* Render the `cosMetrics` prompt source: this install's per-task-type agent RUN stats,
* plus — when the caller has outcome records — the approval → DELIVERY block (#3085).
* Pure; the single place the document's shape is decided, so gatherSources' first pass
* and the hook's delivery-aware re-render can never produce different shapes.
*
* The delivery keys are serialized FIRST so the `COS_METRICS_MAX_CHARS` cap can never
* truncate them away: an install with dozens of task types can fill the budget on its
* own, and the per-task-type tail is the half that already has an interpreted sibling
* in the prompt (liScopeAwareness, derived server-side from the untruncated map),
* whereas the delivery block has none.
*
* `delivery` is OMITTED entirely (not zeroed) when no outcome records were gathered —
* the outcomes source is off, the tracker cannot report outcomes, or the store was
* unreadable. A gathered-but-empty history still emits the block with a null
* `currentDeliveryRate`, so "we have no delivery data" stays distinguishable from
* "nothing has been approved yet".
*
* Returns '' when there is nothing at all to report (no task types AND no delivery
* data) so the caller omits the source rather than injecting a hollow `{}` block.
* Within the `cosMetrics` source the two halves are independently optional: a fresh
* install with no CoS run history can still have approved-but-undelivered proposals
* worth surfacing, and vice versa. Emitting the document at all still requires the
* `cosMetrics` source itself to be enabled — an app that turned it off gets no
* `### cosMetrics` block, delivery half included, and reads its delivery signal from
* the `liOutcomes` / `liSelfEval` blocks instead.
*
* @param {object} args
* @param {Object} [args.metricsByType] - per-task-type run stats (see gatherSources).
* @param {{ delivery: object, deliveryByScope: object }|null} [args.delivery] - a
* `computeDeliveryMetrics(outcomes)` result, or null when none was gathered.
* @returns {string} the JSON document capped at COS_METRICS_MAX_CHARS, or ''.
*/
export function renderCosMetricsSource({ metricsByType = {}, delivery = null } = {}) {
if (!delivery && Object.keys(metricsByType || {}).length === 0) return '';
const payload = {};
// Reserved keys are INSERTED first (JSON.stringify follows insertion order, so the
// cap above can't cut them) but ASSIGNED last, so a task type that happens to share
// one of their names can't shadow the delivery block.
if (delivery) for (const key of COS_METRICS_DELIVERY_KEYS) payload[key] = null;
Object.assign(payload, metricsByType);
if (delivery) {
payload.delivery = delivery.delivery;
payload.deliveryByScope = delivery.deliveryByScope;
}
return JSON.stringify(payload).slice(0, COS_METRICS_MAX_CHARS);
}
const CHURN_SIGNAL_MAX_ITEMS = 8;
const CHURN_SIGNAL_MAX_CHARS = 4000;
/**
* Render the active churn signals for the PortOS install. These are local
* diagnostic measurements, not tracker work: the reasoner must investigate
* them and decide whether a concrete planned fix is justified. Pure.
*/
export function renderChurnSignals(metricsByType, { maxItems = CHURN_SIGNAL_MAX_ITEMS } = {}) {
const signals = Object.entries(metricsByType || {})
.filter(([, metrics]) => metrics?.churn && typeof metrics.churn === 'object')
.slice(0, Math.max(0, maxItems));
if (!signals.length) return '';
const lines = [
'Instance-local CoS churn signals (deterministic telemetry; not tracker issues):',
'These measurements describe this PortOS install only. Investigate whether each signal is a reproducible defect with a concrete planned fix; do not file a proposal merely because a signal exists. Return proposal: null when the evidence does not support actionable work.'
];
for (const [type, metrics] of signals) {
const churn = metrics.churn;
const label = String(type).replace(/\s+/g, ' ').slice(0, 160);
const timing = churn.shortLivedSampleSize > 0
? `${churn.shortLivedCount}/${churn.shortLivedSampleSize} timed runs under ${SHORT_LIVED_MS}ms`
: 'completion spacing only (no per-run durations)';
lines.push(`- ${label}: signal=${churn.signal}; completions=${churn.windowCompleted}; windowMs=${churn.windowMs}; ${timing}; medianDurationMs=${churn.medianDurationMs ?? 'unavailable'}; medianGapMs=${churn.medianGapMs ?? 'unavailable'}`);
}
const total = Object.values(metricsByType || {})
.filter(metrics => metrics?.churn && typeof metrics.churn === 'object')
.length;
if (total > signals.length) lines.push(`- ${total - signals.length} additional churn signal(s) omitted from this prompt for brevity.`);
return lines.join('\n').slice(0, CHURN_SIGNAL_MAX_CHARS);
}
/**
* Gather the enabled Layer-1 sources for one app into a `{ key: string }` map.
* Deterministic reads only (files + CoS metric JSON + tracker lists); NO LLM calls.
* Missing files degrade to omitted keys, never throws. `openIssues` is gathered
* separately by the handler (it shells out to the forge). `tracker`
* (`{ filer, forgeCli, cwd, jira }`, resolved by the caller) enables the
* plannedWork source — absent, that source is simply skipped.
*/
export async function gatherSources(app, config, { cosPath = PATHS.cos, trustShellSources, tracker = null, isPortos = false } = {}) {
const out = {};
const src = config.sources || {};
const repo = app.repoPath;
// Resolve the install-level shell-trust opt-in lazily and once — only when a
// `cmd` source is actually present — so apps with no shell sources never read
// settings. Injected value (tests) wins; otherwise fall back to settings.json.
let trustShell = trustShellSources;
const resolveTrustShell = async () => {
if (trustShell === undefined) trustShell = await getTrustShellSources();
return trustShell;
};
if (src.goals && repo) {
const goals = await tryReadFile(join(repo, 'GOALS.md'));
if (goals) out.goals = goals.slice(0, 8000);
}
if (src.appMetrics && repo) {
// The app's own success/performance metrics doc (the METRICS.md convention,
// see docs/METRICS.md) — where a managed app records what "performing well"
// means. Absent → omitted (the reasoner may then propose adding one).
const metrics = await tryReadFile(join(repo, 'METRICS.md'));
if (metrics) out.appMetrics = metrics.slice(0, 8000);
}
if (src.productMetrics && isPortos) {
// Product-success signals come from PortOS's local stores rather than a
// repo document. Preserve an unavailable sentinel on read failure so LI
// cannot mistake a failed read for zero engagement.
out.productMetrics = await import('../portosProductMetrics.js')
.then(({ getProductMetricsSource }) => getProductMetricsSource())
.catch(() => JSON.stringify({ status: 'unavailable', reason: 'product-metrics-read-failed' }));
}
if (src.planMd && repo) {
const plan = await tryReadFile(join(repo, 'PLAN.md'));
if (plan) out.planMd = plan.slice(0, 8000);
}
if (src.healthReport && repo) {
const health = await tryReadFile(join(repo, 'HEALTH_REPORT.md'));
if (health) out.healthReport = health.slice(0, 8000);
}
if (src.plannedWork && tracker) {
// The committed backlog (#2698). Unlike the file sources above, an EMPTY or
// FAILED result still emits a key — each renders a distinct sentence, and
// both are meaningful instructions to the reasoner (see gatherPlannedWork).
const planned = await gatherPlannedWork(tracker);
if (planned) out.plannedWork = planned.slice(0, PLANNED_WORK_MAX_CHARS);
}
if (src.cosMetrics) {
// This install's own autonomous-agent run stats (per task type), NOT scoped to
// the app being analyzed — see the default-config note for the PortOS-vs-managed
// rationale (default-off for managed apps).
const learning = await readJSONFile(join(cosPath, 'learning.json'), null);
// Surface BOTH the lifetime rate (the cumulative dashboard/telemetry truth)
// AND a recency-windowed rate (issue #2460) per task type, labeled distinctly
// so the reasoner doesn't conflate them. The windowed rate lets a
// since-resolved failure burst age out of the "is work needed" signal instead
// of permanently depressing it; `recentSuccessRate` is null when there are no
// in-window runs, in which case the reasoner leans on the lifetime rate.
// Note the intentional rename: computeWindowedStats' internal `windowed*`
// fields are surfaced to the reasoner as `recent*` (reads more naturally in
// the prompt context) — same concept, deliberately different label here.
//
// An unreadable / byTaskType-less learning store yields an EMPTY map rather than
// skipping the source: the delivery half of this document (#3085) comes from the
// outcome records, not learning.json, so an install with no CoS run history can
// still have approved-but-undelivered proposals worth surfacing. When both halves
// are empty renderCosMetricsSource returns '' and the source is omitted as before.
const summary = {};
const DAY_MS = 24 * 60 * 60 * 1000;
for (const [type, m] of Object.entries(learning?.byTaskType || {})) {
const windowed = computeWindowedStats(m?.recentOutcomes);
// 24h burst stats (duration + count) plus the deterministic churn signal
// below so a future PortOS LI run can judge whether the local evidence
// supports a concrete planned fix.
const burst = summarizeRecentRuns(m?.recentOutcomes, { windowMs: DAY_MS });
const entry = {
lifetimeSuccessRate: typeof m?.successRate === 'number' ? m.successRate : null,
lifetimeCompleted: m?.completed || 0,
recentSuccessRate: windowed.windowedSuccessRate,
recentCompleted: windowed.windowedCompleted,
avgDurationMs: m?.avgDurationMs || 0,
recent24hCompleted: burst.windowCompleted,
recent24hShortLived: burst.shortLivedCount,
recent24hMedianDurationMs: burst.medianDurationMs,
shortLivedMs: SHORT_LIVED_MS
};
const churn = isPortos ? computeChurn(m?.recentOutcomes) : null;
if (churn?.flagged) {
entry.churn = {
signal: churn.reason,
windowMs: churn.windowMs,
windowCompleted: churn.windowCompleted,
shortLivedCount: churn.shortLivedCount,
shortLivedSampleSize: churn.shortLivedSampleSize,
shortLivedRatio: churn.shortLivedRatio,
medianDurationMs: churn.medianDurationMs,
medianGapMs: churn.medianGapMs
};
}
summary[type] = entry;
}
// No delivery block yet: the approval → delivery numbers come from the app's
// outcome records, which the hook only has AFTER it has reconciled them against
// this run's tracker read. The raw per-type map rides along on `cosMetricsByType`
// so the hook can re-render the document once with both halves (see
// renderCosMetricsSource) rather than emitting a delivery figure one cycle stale —
// two blocks in one prompt disagreeing about how many proposals are approved is
// worse than either number alone.
out.cosMetricsByType = summary;
const rendered = renderCosMetricsSource({ metricsByType: summary });
if (rendered) out.cosMetrics = rendered;
// Scope-awareness guidance (#2760): a deterministic low/high-completion split
// derived from the SAME per-type rates above, so the reasoner gets an interpreted
// signal alongside the raw JSON instead of being asked to spot the pattern itself.
// Rendered as its own prompt block (see buildPrompt). Gated on isPortos (codex P2):
// these are THIS install's own CoS completion rates, meaningless to a managed app —
// and a managed app CAN enable the cosMetrics source, so the cosMetrics toggle
// alone is not the PortOS boundary. buildPrompt re-checks isPortos as defense in
// depth; deriving it here only for PortOS also avoids the wasted work.
if (isPortos) {
const scopeGuidance = computeScopeAwareness({ metricsByType: summary });
if (scopeGuidance) out.scopeGuidance = scopeGuidance;
const churnSignals = renderChurnSignals(summary);
if (churnSignals) out.churnSignals = churnSignals;
}
}
for (const custom of src.custom || []) {
const key = customSourceKey(custom);
if (!key) continue;
if (custom.type === 'file' && typeof custom.ref === 'string' && repo) {
const safe = await confineToRepo(repo, custom.ref);
if (!safe) {
console.warn(`⚠️ Layered Intelligence: custom source "${custom.ref}" escapes repo — skipped`);
continue;
}
const content = await tryReadFile(safe);
if (content) out[key] = content.slice(0, 8000);
} else if (custom.type === 'http' && typeof custom.url === 'string') {
const content = await fetchHttpSource(custom.url);
if (content) out[key] = content.slice(0, 8000);
} else if (custom.type === 'cmd' && typeof custom.cmd === 'string' && repo) {
const content = await runShellCommand(custom.cmd, { cwd: repo, trustShellSources: await resolveTrustShell() });
if (content) out[key] = content.slice(0, 8000);
}
}
return out;
}
/**
* Read the LI loop's OWN agent-run metrics out of the CoS learning store (#2700).
* Deterministic file read; no LLM call. Feeds computeSelfEvalSummary's execution-
* health signal.
*
* Returns a discriminated result rather than a bare bucket-or-null, because the two
* empty cases are NOT the same fact and the reasoner is told them differently:
* `{ read: false, metrics: null }` — the store is missing/unreadable/malformed:
* we do not know how LI's runs are going.
* `{ read: true, metrics: null }` — the store is fine, LI has simply never run.
* `{ read: true, metrics: {...} }` — real history.
* Collapsing these to one `null` would let "cannot read the store" masquerade as
* "healthy loop with no history" (or vice versa) — the sentinel rule.
*/
export async function readLiTaskMetrics({ cosPath = PATHS.cos } = {}) {
const file = join(cosPath, 'learning.json');
// An ABSENT store is a fresh install, not a broken read: learning.json is created
// lazily on the first recorded task outcome. readJSONFile returns its default for
// ENOENT, I/O errors, and parse failures ALIKE, so leaning on it alone would tell
// every fresh install "your learning store could not be read" when the truth is
// "nothing has run here yet" — the exact conflation this function exists to
// prevent, just inverted. Check existence first so the two stay distinct.
if (!existsSync(file)) return { read: true, metrics: null };
const learning = await readJSONFile(file, null);
const byTaskType = learning?.byTaskType;
if (!byTaskType || typeof byTaskType !== 'object' || Array.isArray(byTaskType)) {
return { read: false, metrics: null };
}
const bucket = byTaskType[LI_TASK_TYPE];
return {
read: true,
metrics: (bucket && typeof bucket === 'object' && !Array.isArray(bucket)) ? bucket : null
};
}
/**
* Stable map key for a custom source. Namespaced by type so a `file` ref and an
* `http` url that share a string can't collide, and so the prompt's source block
* labels are self-describing. Returns null for a malformed/unknown source.
*/
export function customSourceKey(custom) {
if (!custom || typeof custom !== 'object') return null;
if (custom.type === 'file' && custom.ref) return `custom:${custom.ref}`;
if (custom.type === 'http' && custom.url) return `custom:http:${custom.url}`;
if (custom.type === 'cmd' && custom.cmd) return `custom:cmd:${custom.cmd}`;
return null;
}
/**
* Fetch an http(s) custom source for the loop's prompt. Deterministic read, no
* LLM. Rejects any non-http(s) scheme (defense in depth over the Zod refine),
* bounds the request with a 10s timeout, and returns null on any failure so a
* dead URL just omits the key rather than throwing.
*
* SSRF-guarded via `fetchPublicText` (default posture): loopback, link-local,
* and the cloud-metadata endpoint (127.0.0.1, 169.254.169.254,
* metadata.google.internal, ::1) are blocked so a hand-edited/hostile config
* can't exfiltrate them into the reasoner prompt, and redirects are revalidated
* against the same gate. LAN/private hosts (Tailscale peers, a home wiki) stay
* ALLOWED intentionally — PortOS is a single-user tool where a custom source
* legitimately points at the home network, and the URL is operator-configured.
* `throwOnUnsafe: false` makes a blocked host omit the key like any other dead
* URL instead of bubbling a 400. `fetchText` is injectable for tests.
*/
export async function fetchHttpSource(url, { timeoutMs = 10_000, fetchText = fetchPublicText } = {}) {
if (typeof url !== 'string' || !/^https?:\/\//i.test(url)) return null;
const text = await fetchText(url, { timeoutMs, throwOnUnsafe: false }).catch(() => null);
return text || null;
}
/**
* THREAT MODEL — a `cmd` custom source is attacker-reachable persistent config.
*
* The `sources.custom` array lives in each app's stored `layeredIntelligence`
* config, written through the validated `PUT /api/apps/:id` route. A `cmd` entry
* is executed here on the Layered Intelligence SCHEDULE (Engine-B autonomous job),
* with the PortOS process's own privileges and cwd = the app repo. So any path
* that can land a string in that config (a hand-edited config, a hostile sync
* payload, a future config-writing feature, an XSS-driven same-origin POST) gets
* *persistent, unattended* code execution — not a one-shot the operator watched.
*
* Historically this ran the full command string with `shell: true`, capped only
* by length + a 15s timeout. That is arbitrary RCE: `; rm -rf ~`, `$(curl … | sh)`,
* pipes to `sh`, etc. all execute. Issue #2515.
*
* Defense: by default we DENY the shell. The command is parsed and checked by
* `validateUnattendedCommand` (commandSecurity.js), which rejects shell
* metacharacters (`;|&$(){}` …) and admits only the read-only inspection
* binaries in `UNATTENDED_READONLY_COMMANDS` (`git`, `gh`, `glab`, `ls`, `cat`,
* `head`, `tail`, `grep`, `find`, `wc`, `pwd`, `echo`); then we spawn the base
* binary with parsed args and `shell: false` — so no shell ever interprets the
* string. A rejected command is dropped (key omitted) with a warning, exactly
* like any other failed source read.
*
* This lane deliberately does NOT use the operator-facing `validateCommand`
* allowlist. That one admits `npx`, `node`, `python`, `pip`, `curl`, `wget`,
* `go`, `cargo`, `make` and `brew` — every one of which fetches and/or executes
* arbitrary code without a single shell metacharacter (`npx <pkg>`,
* `pip install <pkg>`, `curl -o <path> <url>`), so the metacharacter filter
* would be doing all the work here. Fine for a run a human triggered and is
* watching; not fine for persistent config on an autonomous schedule. Issue #5669.
*
* Escape hatch: an operator who genuinely needs a pipeline (`git log … | head`)
* or a binary outside the read-only list can set the install-level
* `settings.layeredIntelligence.trustShellSources` flag, which restores the full
* `shell: true` behavior for THIS install only. It is an explicit, install-wide
* opt-in — off by default — so a fresh install (or a synced-in app config) can
* never execute anything but a read-only inspection command.
*
* `exec` is injectable for tests; `trustShellSources` is resolved by the caller
* (`gatherSources`) from install settings and threaded in.
*
* Returns null on rejection / non-zero exit / timeout / no output so a failing or
* denied command just omits the source key rather than throwing.
*/
export async function runShellCommand(cmd, { cwd, timeoutMs = 15_000, exec = bufferedSpawn, trustShellSources = false } = {}) {
if (typeof cmd !== 'string' || !cmd.trim()) return null;
if (trustShellSources) {
// Operator has explicitly opted this install into full-shell custom sources.
const { code, stdout } = await exec(cmd, [], { cwd, timeoutMs, shell: true });
if (code !== 0) return null;
return (stdout || '').trim() || null;
}
const check = validateUnattendedCommand(cmd);
if (!check.valid) {
console.warn(`⚠️ Layered Intelligence: custom cmd source "${cmd}" rejected — ${check.error} (unattended sources are limited to read-only inspection commands; enable settings.layeredIntelligence.trustShellSources to allow arbitrary shell commands)`);
return null;
}
const { code, stdout } = await exec(check.baseCommand, check.args, { cwd, timeoutMs, shell: false });
if (code !== 0) return null;
return (stdout || '').trim() || null;
}
/**
* Resolve the install-level "trust shell sources" opt-in from settings.json.
* `null`/absent/non-true all read as OFF (the safe default) — only an explicit
* `true` unlocks full-shell custom `cmd` sources. Injectable read for tests.
*/
export async function getTrustShellSources(read = getSettings) {
const settings = await read();
return settings?.layeredIntelligence?.trustShellSources === true;
}
/**
* Confine a custom file `ref` to within `repo` so a hostile/hand-edited config
* can't read arbitrary files into the LLM prompt. Returns the safe absolute path,
* or null when it escapes. Guards BOTH lexical traversal (`..` / absolute) AND
* symlink escape — a symlink inside the repo pointing outside is resolved via
* realpath and rejected. Missing files return null (nothing to read).
*/
export async function confineToRepo(repo, ref) {
const abs = resolve(repo, ref);
const rel = relative(repo, abs);
if (rel.startsWith('..') || isAbsolute(rel)) return null;
// Resolve symlinks on both sides; a link inside the repo that points outside
// is caught here (lexical check above only sees the link's own path).
const realRepo = await realpath(repo).catch(() => null);
const realAbs = await realpath(abs).catch(() => null);
if (!realRepo || !realAbs) return null;
const realRel = relative(realRepo, realAbs);
if (realRel.startsWith('..') || isAbsolute(realRel)) return null;
return realAbs;
}