Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Shared JSON extraction utilities for LLM responses.
*
* LLM output is messy — CLI providers (notably Codex) prepend banner text
* and echo the user prompt back to stdout before the model response. The
* prompt itself often contains a JSON-shaped schema example whose braces
* balance but whose contents are not valid JSON. Models also routinely
* emit trailing commas, `[...]` placeholder elisions, and the Codex
* `}}]` orphan-brace corruption pattern.
*
* Callers share the brace walkers and repair passes; refinement callers
* additionally reject echoed schema placeholders with extractNonPlaceholderJson.
*/
import { stripCodeFences } from './llmText.js';
/**
* Walk the string and return every top-level brace-balanced block, in
* order. String-aware so braces/brackets inside JSON string values don't
* throw off the depth counter. Returning every block lets the caller try
* each in turn — preferring the one whose shape matches the expected
* response over an in-prompt schema example.
*
* @param {string} s — input text
* @param {object} [options]
* @param {string} [options.startChar='{'] — opening delimiter ('{' or '[')
* @param {string} [options.endChar='}'] — matching closing delimiter
* @returns {string[]} — every balanced block found, in source order
*/
export function findBalancedBlocks(s, { startChar = '{', endChar = '}' } = {}) {
if (typeof s !== 'string' || !s) return [];
const blocks = [];
let i = 0;
while (i < s.length) {
const start = s.indexOf(startChar, i);
if (start === -1) break;
let depth = 0;
let inString = false;
let escaped = false;
let end = -1;
for (let j = start; j < s.length; j += 1) {
const ch = s[j];
if (inString) {
if (escaped) escaped = false;
else if (ch === '\\') escaped = true;
else if (ch === '"') inString = false;
continue;
}
if (ch === '"') { inString = true; continue; }
if (ch === startChar) depth += 1;
else if (ch === endChar) {
depth -= 1;
if (depth === 0) { end = j; break; }
}
}
// Intentional bail on the first unbalanced segment. A well-formed
// block COULD exist further along (e.g. an LLM that dumped partial
// JSON, then prose, then a complete second JSON object), but for
// PortOS's LLM-output use case the more common shape is "model
// emitted one JSON and then trailed off" — surfacing the candidates
// we have so far and letting the caller decide is safer than gluing
// together fragments that span an unbalanced gap. If a future use
// case needs scan-past-unbalanced behavior, add an opt-in option.
if (end === -1) break;
blocks.push(s.slice(start, end + 1));
i = end + 1;
}
return blocks;
}
/**
* Every brace-balanced block in the string — nested ones included — in the
* order they CLOSE. String-aware like `findBalancedBlocks`, but built on a
* stack instead of a restart-from-scratch scan, which buys two properties the
* transcript rescue in `agentSentinel.extractSentinelPayloadFromTranscript`
* needs and `findBalancedBlocks` deliberately does not provide:
*
* 1. An unmatched opening brace does NOT end the walk. `findBalancedBlocks`
* bails at the first unbalanced segment (see its note); a repaint-heavy
* PTY transcript is full of stray braces from truncated redraws, and
* bailing there would discard the complete JSON object the model printed
* at the very end — the whole point of the scan.
* 2. Closing order means an enclosing object is emitted AFTER the objects it
* contains, so a caller walking the result in REVERSE sees the outermost
* object of the last-printed block before any of its children.
*
* Still one linear pass (an unmatched `}` with an empty stack is dropped), so
* scanning a large tail stays cheap. String tracking resets at every line break
* — see the note in the loop for why a transcript needs that.
*
* @param {string} s — input text
* @param {object} [options]
* @param {string} [options.startChar='{'] — opening delimiter
* @param {string} [options.endChar='}'] — matching closing delimiter
* @returns {string[]} — every balanced block, in the order they close
*/
export function findAllBalancedBlocks(s, { startChar = '{', endChar = '}' } = {}) {
if (typeof s !== 'string' || !s) return [];
const blocks = [];
const opens = [];
let inString = false;
let escaped = false;
for (let i = 0; i < s.length; i += 1) {
const ch = s[i];
// A line break ends string tracking. JSON forbids a raw newline inside a
// string literal, so nothing valid is lost — but a transcript is full of
// stray quotes (a truncated redraw, an echoed `echo "…`, a log message),
// and without this ONE of them would leave the walker "inside a string"
// for the rest of the scan, hiding every brace in the JSON that follows.
if (ch === '\n' || ch === '\r') { inString = false; escaped = false; continue; }
if (inString) {
if (escaped) escaped = false;
else if (ch === '\\') escaped = true;
else if (ch === '"') inString = false;
continue;
}
if (ch === '"') { inString = true; continue; }
if (ch === startChar) opens.push(i);
else if (ch === endChar && opens.length) blocks.push(s.slice(opens.pop(), i + 1));
}
return blocks;
}
/**
* Apply a regex `replace()` to the input ONLY OUTSIDE quoted JSON
* string regions. Walks the input with the same string/escape awareness
* as `findBalancedBlocks`, splits into alternating "code" and "string"
* segments, runs the replace on each code segment, then reassembles.
* Strings are passed through verbatim, so a JSON string containing
* `,}` or `}}]` as content is preserved.
*/
function replaceOutsideStrings(input, pattern, replacement) {
if (typeof input !== 'string' || !input) return input;
const segments = [];
let cursor = 0;
let inString = false;
let escaped = false;
for (let i = 0; i < input.length; i += 1) {
const ch = input[i];
if (inString) {
if (escaped) escaped = false;
else if (ch === '\\') escaped = true;
else if (ch === '"') {
// String just closed at index `i`. The open-quote → close-quote
// range (inclusive) is a 'string' segment the replace can't touch.
segments.push({ kind: 'string', text: input.slice(cursor, i + 1) });
cursor = i + 1;
inString = false;
}
continue;
}
if (ch === '"') {
// About to enter a string at index `i`. Flush the preceding code
// region so the replace can fire on it.
if (i > cursor) segments.push({ kind: 'code', text: input.slice(cursor, i) });
cursor = i;
inString = true;
}
}
// Flush trailing region. If the input ended mid-string (malformed JSON),
// treat the unterminated tail as a string so the repair doesn't touch
// it — the parse will fail anyway and the caller falls through.
if (cursor < input.length) {
segments.push({ kind: inString ? 'string' : 'code', text: input.slice(cursor) });
}
return segments.map((seg) => (
seg.kind === 'code' ? seg.text.replace(pattern, replacement) : seg.text
)).join('');
}
/**
* Escape raw ASCII control characters (U+0000–U+001F) that appear INSIDE a
* JSON string literal — the JSON spec forbids them there, they must be
* `\n`/`\t`/`\uXXXX` escapes. Less-capable models (notably local ones asked
* for a large multi-line markdown field) routinely paste literal newlines/tabs
* into a string value, which makes `JSON.parse` throw "Bad control character
* in string literal". String-aware: control chars OUTSIDE strings are the
* structural whitespace between tokens and are left untouched.
*/
function escapeControlCharsInStrings(input) {
if (typeof input !== 'string' || !input) return input;
let out = '';
let inString = false;
let escaped = false;
for (let i = 0; i < input.length; i += 1) {
const ch = input[i];
if (inString) {
const code = input.charCodeAt(i);
// A raw control char is ALWAYS illegal inside a JSON string — it can
// never be the target of a valid `\`-escape (escape targets are the
// printable set `"\/bfnrtu`). So escape it BEFORE consulting `escaped`:
// a control char that follows a backslash means that backslash was a
// dangling/invalid escape, and copying the control char through (the old
// `escaped` branch) left the string unparseable. Resetting `escaped`
// consumes any pending (spurious) escape.
if (code <= 0x1f) {
if (ch === '\n') out += '\\n';
else if (ch === '\r') out += '\\r';
else if (ch === '\t') out += '\\t';
else if (ch === '\b') out += '\\b';
else if (ch === '\f') out += '\\f';
else out += `\\u${code.toString(16).padStart(4, '0')}`;
escaped = false;
continue;
}
if (escaped) { out += ch; escaped = false; continue; }
if (ch === '\\') { out += ch; escaped = true; continue; }
if (ch === '"') { out += ch; inString = false; continue; }
out += ch;
continue;
}
if (ch === '"') { out += ch; inString = true; continue; }
out += ch;
}
return out;
}
/**
* Close a string whose final quote was omitted immediately before a structural
* `}` / `]` line. A raw line break is illegal inside JSON strings; when its next
* non-horizontal-whitespace character closes the containing object or array,
* the only structurally coherent interpretation is that the prose value ended
* before the line break. Other raw newlines remain untouched for the existing
* control-character escape repair.
*/
function closeStringBeforeStructuralLine(input) {
if (typeof input !== 'string' || !input) return input;
let out = '';
let inString = false;
let escaped = false;
for (let i = 0; i < input.length; i += 1) {
const ch = input[i];
if (!inString) {
out += ch;
if (ch === '"') inString = true;
continue;
}
if (escaped) {
out += ch;
escaped = false;
continue;
}
if (ch === '\\') {
out += ch;
escaped = true;
continue;
}
if (ch === '"') {
out += ch;
inString = false;
continue;
}
if (ch === '\n' || ch === '\r') {
let next = i + 1;
while (next < input.length && (input[next] === ' ' || input[next] === '\t')) next += 1;
if (input[next] === '}' || input[next] === ']') {
out += '"';
inString = false;
}
}
out += ch;
}
return out;
}
/**
* Escape a quote that appears inside a JSON string without its required
* backslash. A quote can close a JSON string only when the next non-whitespace
* character is a structural delimiter (`:`, `,`, `}`, or `]`) or the document
* ends. Anything else means prose continues after the quote, so retaining it as
* a delimiter cannot produce valid JSON. This deliberately repairs only that
* unambiguous case; the caller still accepts the result only after a full parse.
*/
function escapeBareQuotesInStrings(input) {
if (typeof input !== 'string' || !input) return input;
let out = '';
let inString = false;
let escaped = false;
for (let i = 0; i < input.length; i += 1) {
const ch = input[i];
if (!inString) {
out += ch;
if (ch === '"') inString = true;
continue;
}
if (escaped) {
out += ch;
escaped = false;
continue;
}
if (ch === '\\') {
out += ch;
escaped = true;
continue;
}
if (ch !== '"') {
out += ch;
continue;
}
let next = i + 1;
while (next < input.length && /\s/.test(input[next])) next += 1;
if (next >= input.length || /[:,}\]]/.test(input[next])) {
out += ch;
inString = false;
} else {
out += '\\"';
}
}
return out;
}
/**
* Try JSON.parse on a candidate block. If it fails, apply cheap repairs
* for observed LLM corruption patterns and try again:
* - Trailing commas before `}` or `]` (common LLM mistake).
* - `[...]` literal placeholder elisions echoed from prompt examples.
* - Codex CLI `}}]` orphan-brace corruption — an extra `}` snuck in
* between a variation's close-brace and the array's `]`. Swapping
* `}}]` → `}]}` (not dropping the brace) keeps the brace count
* correct so the outer container still closes.
* - A serialized JSON tail — the model begins with ordinary JSON, then
* accidentally emits the remainder as JSON-string escapes (`\\n`, `\\"`).
* Decode only from JSON.parse's exact failure position and only keep the
* result when the reconstructed whole document parses.
* - Unescaped quotation marks inside prose string values — escape only a
* quote that cannot legally close the string because prose follows it.
* - A missing closing quote immediately before a structural `}` / `]` line.
* - Raw control chars (literal newlines/tabs) inside a string value —
* escaped to their `\n`/`\t`/`\uXXXX` forms (see
* escapeControlCharsInStrings). Common when a model writes a long
* multi-line markdown field into a string literal verbatim.
*
* The brace/comma repairs are STRING-AWARE: the regexes only touch
* characters outside quoted JSON string values, so a string containing
* `,}` or `}}]` as ordinary content is preserved. The control-char repair
* is likewise confined to inside-string regions.
*
* Returns `{ value }` on success (where `value` may be a valid JSON
* `null`); returns `{ error }` if all repairs fail. The wrapper lets a
* top-level `null` response flow through as a real parsed value rather
* than colliding with a "did not parse" sentinel.
*
* @param {string} jsonText — candidate JSON text
* @returns {{ value: unknown } | { error: Error }}
*/
export function tryParseWithRepair(jsonText) {
if (typeof jsonText !== 'string') return { error: new Error('Non-string input') };
// `[...]` placeholder cleanup runs before the first parse so a block
// containing only that token (no other JSON errors) succeeds on the
// first try instead of falling into the trailing-comma branch.
const initial = replaceOutsideStrings(jsonText, /\[\s*\.\.\.\s*\]/g, '[]');
const initialResult = safeParse(initial);
if (!initialResult.error) return initialResult;
const unescapedTail = unescapeSerializedJsonTail(initial, initialResult.error);
if (unescapedTail !== initial) {
const tailResult = safeParse(unescapedTail);
if (!tailResult.error) return tailResult;
}
const closedStrings = closeStringBeforeStructuralLine(unescapedTail);
if (closedStrings !== unescapedTail) {
const closedResult = safeParse(closedStrings);
if (!closedResult.error) return closedResult;
}
const escapedQuotes = escapeBareQuotesInStrings(closedStrings);
if (escapedQuotes !== closedStrings) {
const quotesResult = safeParse(escapedQuotes);
if (!quotesResult.error) return quotesResult;
}
const noTrailing = replaceOutsideStrings(escapedQuotes, /,(\s*[}\]])/g, '$1');
if (noTrailing !== escapedQuotes) {
const trailingResult = safeParse(noTrailing);
if (!trailingResult.error) return trailingResult;
}
const fixedOrphan = replaceOutsideStrings(noTrailing, /}\s*}\s*]/g, '}]}');
const orphanResult = safeParse(fixedOrphan);
if (!orphanResult.error) return orphanResult;
// Last resort: escape raw control chars inside string literals. Runs on top
// of the prior repairs (cumulative) so trailing-comma + orphan-brace + raw
// newline corruption in one block still recovers. safeParse already returns
// the discriminated { value }/{ error } shape, so pass it straight through.
const escaped = escapeControlCharsInStrings(fixedOrphan);
if (escaped !== fixedOrphan) return safeParse(escaped);
// Surface the last attempt's parse error so the caller can include the
// concrete reason ("Unexpected token } in JSON at position 47") in its
// error message rather than a generic "no JSON block found".
return { error: orphanResult.error };
}
/**
* Recover an observed one-shot LLM corruption where a valid JSON prefix is
* followed by the rest of the document encoded as a JSON string fragment.
* V8 reports the first outside-string backslash as the parse position, giving
* us an exact and conservative decode boundary. Wrapping the tail in quotes
* decodes its `\\n` / `\\"` sequences without guessing at individual escapes;
* callers still require the entire reconstructed document to parse before
* accepting it.
*/
function unescapeSerializedJsonTail(input, error) {
const match = error?.message?.match(/position (\d+)/);
const position = Number(match?.[1]);
if (!Number.isInteger(position) || position < 0 || input[position] !== '\\') return input;
const tail = input.slice(position);
if (!/^\\(?:["\\/bfnrt]|u[0-9a-fA-F]{4})/.test(tail)) return input;
const trailingWhitespace = tail.match(/\s*$/)?.[0] || '';
const encodedTail = trailingWhitespace ? tail.slice(0, -trailingWhitespace.length) : tail;
const decoded = safeParse(`"${encodedTail}"`);
if (decoded.error || typeof decoded.value !== 'string') return input;
return `${input.slice(0, position)}${decoded.value}${trailingWhitespace}`;
}
// Returns `{ value }` on success or `{ error }` on failure. The discriminated
// shape lets a JSON `null` literal flow through as a valid parsed value
// (some LLMs legitimately return `null`); a single-value-with-sentinel
// API can't distinguish "parsed `null`" from "did not parse."
function safeParse(text) {
try { return { value: JSON.parse(text) }; } catch (error) { return { error }; }
}
/**
* Extract the first matching JSON block from CLI-banner-prefixed LLM
* output. Strips ```json / ``` fences, walks balanced blocks, applies
* repairs, and returns the first block matching the optional shape
* predicate (or the first block that parses at all when no predicate
* is supplied).
*
* Returns `{ value: undefined, lastError, lastPreview }` if no block
* parses — callers decide whether to throw a typed error (ServerError)
* or attempt a different fallback.
*
* @param {string} text — raw LLM output
* @param {object} [options]
* @param {(parsed:unknown)=>boolean} [options.shapePredicate] — return true
* for blocks whose shape matches the caller's expected response. Used to
* skip in-prompt schema examples that parse cleanly but aren't the answer.
* @param {'object'|'array'} [options.blockType='object'] — top-level shape
* to walk for (`{...}` vs `[...]`).
* @param {boolean} [options.skipInnerFence=false] — skip the "first inner
* ```…``` block" heuristic (see below). Set this when the JSON's own string
* VALUES may contain fenced markdown (e.g. a proposal `body` with a code
* snippet): the heuristic would otherwise lock onto that inner fence and
* discard the surrounding object. Only the outer-wrapper strip
* (`stripCodeFences`) runs, then balanced-block walking — which is
* string-aware, so backticks inside a JSON string can't derail it.
* @returns {{ value:unknown, lastError?:Error, lastPreview?:string }}
* — `value` is the parsed block, or `undefined` when no block matches.
* On no-match, `lastError` + `lastPreview` (200-char excerpt of the last
* candidate text) are populated for use in caller error messages.
*/
export function extractJson(text, { shapePredicate, blockType = 'object', skipInnerFence = false } = {}) {
if (typeof text !== 'string' || !text.trim()) {
return { value: undefined, lastError: new Error('Empty LLM response'), lastPreview: '' };
}
let s = stripCodeFences(text.trim());
// stripCodeFences only catches leading/trailing fences. CLI banners
// sometimes wrap the response in fences mid-stream — fall back to
// a non-greedy match for the FIRST inner ```…``` block so we still
// extract the inner JSON. NOTE: callers that may see fenced
// prompt-echo content BEFORE the real response should skip this
// helper and walk the full text directly (see stageRunner.extractJson)
// — the first-fence heuristic locks onto the echoed schema and never
// reaches the actual model output. skipInnerFence disables it for callers
// whose JSON string values legitimately contain fenced markdown (the fence
// is content, not a wrapper) — see the option doc above.
if (!skipInnerFence) {
const fence = s.match(/```(?:json)?\s*([\s\S]*?)```/);
if (fence) s = fence[1].trim();
}
const { startChar, endChar } = blockType === 'array'
? { startChar: '[', endChar: ']' }
: { startChar: '{', endChar: '}' };
const candidates = findBalancedBlocks(s, { startChar, endChar });
if (!candidates.length) candidates.push(s);
// `parsedHolder` carries the first parseable block as `{ value }` even
// when it doesn't match shapePredicate, so a top-level `null` literal
// can still flow through as a real parsed value on fallback.
let parsedHolder;
let lastError;
let lastPreview = s.slice(0, 200);
for (const block of candidates) {
const result = tryParseWithRepair(block);
if (result.error) {
lastError = result.error;
lastPreview = block.slice(0, 200);
continue;
}
if (!shapePredicate || shapePredicate(result.value)) {
return { value: result.value };
}
if (parsedHolder === undefined) parsedHolder = result;
}
if (parsedHolder !== undefined) return parsedHolder;
return {
value: undefined,
lastError: lastError || new Error('No matching JSON block found'),
lastPreview,
};
}
/**
* Extract a refinement with a string field, skipping echoed <...> schema
* placeholders. Keeps placeholder-only failures distinct from parse failures;
* unlike extractJson, never falls back to an object with the wrong shape.
* Preserves the refiners' first-inner-fence heuristic.
*/
export function extractNonPlaceholderJson(raw, { field }) {
if (typeof raw !== 'string' || !raw.trim()) throw new Error('Empty AI response');
let s = raw.trim();
const fence = s.match(/```(?:json)?\s*([\s\S]*?)```/);
if (fence) s = fence[1].trim();
const candidates = findBalancedBlocks(s);
if (!candidates.length) candidates.push(s);
// Walk every balanced block ourselves rather than calling the shared
// extractJson — we need a tri-state outcome (real refinement, placeholder
// echo, or parse error) to surface the "schema placeholder" error message
// that helps users pick a stronger model. extractJson's shape predicate
// collapses placeholder-vs-real into a single fallback path.
let placeholderSeen = false;
let lastErr;
for (const block of candidates) {
const result = tryParseWithRepair(block);
if (result.error) {
lastErr = result.error;
continue;
}
const { value } = result;
if (value && typeof value === 'object' && typeof value[field] === 'string') {
if (/^\s*<.+>\s*$/.test(value[field])) { placeholderSeen = true; }
else return value;
}
}
if (placeholderSeen) {
throw new Error('AI returned the schema placeholder instead of a real refinement — try a stronger model or rerun');
}
throw new Error(`Invalid JSON in AI response${lastErr ? `: ${lastErr.message}` : ''}`);
}