Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Build API-wrapper scan views in one lexer pass (#11561)

[._.]/a983baff59e9
diff --git a/client/src/apiWrapperOptionsConventions.test.js b/client/src/apiWrapperOptionsConventions.test.js
index d9e4e75a7c..3bf08e1886 100644
--- a/client/src/apiWrapperOptionsConventions.test.js
+++ b/client/src/apiWrapperOptionsConventions.test.js
@@ -42,12 +42,22 @@ import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { trackedSourceFiles } from './test/trackedFiles.js';
-import { blankCommentBodies, blankLiterals, matchBracket } from '../../server/lib/sourceScan.js';
+import { blankLiterals, matchBracket } from '../../server/lib/sourceScan.js';
const CLIENT_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const SERVICE_FILE = /^src\/services\/api[^/]*\.js$/;
const OPTIONS_NAMES = new Set(['options', 'opts']);
+// Opt-in stage evidence for workload-sensitive timeouts; normal runs stay quiet.
+const profile = process.env.PORTOS_PROFILE_API_WRAPPERS === '1' ? new Map() : null;
+const measure = (stage, run) => {
+ if (!profile) return run();
+ const start = performance.now();
+ const result = run();
+ profile.set(stage, (profile.get(stage) ?? 0) + performance.now() - start);
+ return result;
+};
+
/** Split `text[start, end)` on top-level commas, using the blanked copy to find them. */
function splitTopLevel(blanked, text, start, end) {
const parts = [];
@@ -71,8 +81,7 @@ const WRAPPER_HEAD = /^[ \t]*export\s+(?:(?:async\s+)?function\s+(\w+)\s*\(|cons
/** Exported function wrappers in one service file: `[{ name, params, text }]`. */
export function wrapperExports(src) {
- const code = blankCommentBodies(src);
- const blanked = blankLiterals(src);
+ const { code, blanked } = measure('lex', () => blankLiterals(src, { includeCommentOnly: true }));
const heads = [...blanked.matchAll(WRAPPER_HEAD)];
const allExports = [...blanked.matchAll(/^[ \t]*export\s/gm)].map((m) => m.index);
return heads.map((head) => {
@@ -122,8 +131,7 @@ const SILENT_KEY = /[{,]\s*silent\s*(?::|,|\})/;
*/
export function silentCalls(src, names) {
if (!src.includes('silent')) return [];
- const code = blankCommentBodies(src);
- const blanked = blankLiterals(src);
+ const { code, blanked } = measure('lex', () => blankLiterals(src, { includeCommentOnly: true }));
const imported = new Set([...code.matchAll(/import\s*\{([^}]*)\}\s*from/g)]
.flatMap((m) => m[1].split(',').map((part) => part.trim().split(/\s+as\s+/).pop())));
const hits = [];
@@ -165,13 +173,17 @@ const read = (file) => ({ file, src: readFileSync(join(CLIENT_ROOT, file), 'utf8
describe('silent options reach the API wrapper that reads them (#9951)', () => {
it('has no { silent } passed to a position the wrapper drops', () => {
- const tracked = trackedSourceFiles(CLIENT_ROOT);
+ profile?.clear();
+ const tracked = measure('discovery', () => trackedSourceFiles(CLIENT_ROOT));
expect(tracked.length).toBeGreaterThan(100);
- const wrappers = tracked.filter((file) => SERVICE_FILE.test(file)).flatMap((file) => wrapperExports(read(file).src));
+ const files = measure('read', () => tracked.map(read));
+ const wrappers = measure('wrappers', () => files.filter(({ file }) => SERVICE_FILE.test(file)).flatMap(({ src }) => wrapperExports(src)));
// A broken parse would otherwise pass by judging nothing.
expect(wrappers.length).toBeGreaterThan(500);
- const violations = findDroppedSilent(tracked.map(read), wrappers);
+ const violations = measure('calls', () => findDroppedSilent(files, wrappers));
+ // Explicit profiling must survive CI's suppression of passing-test consoles.
+ if (profile) process.stdout.write(`⏱ API wrapper scan: files=${files.length}, wrappers=${wrappers.length}, ${[...profile].map(([stage, ms]) => `${stage}=${ms.toFixed(1)}ms`).join(', ')}\n`);
expect(
violations,
'These calls pass `{ silent: … }` to an API wrapper whose parameter at that position does not '
@@ -219,6 +231,31 @@ describe('silent options reach the API wrapper that reads them (#9951)', () => {
expect(silentCalls(`${imp}// noOptions(id, { silent: true })`, names)).toEqual([]);
});
+ it('keeps dynamic endpoints, nested defaults, and calls inside template interpolation visible', () => {
+ const wrappers = wrapperExports(`
+ export const dynamic = (id, data = { label: \`item-\${id}\` }, extra) =>
+ request(\`/things/\${id ? 'nested)' : 'other'}\`, { body: JSON.stringify(data), ...extra });
+ export async function dropped(id = choose({ nested: [1, 2] })) {
+ return request(\`/things/\${id}\`);
+ }
+ `);
+ expect(wrappers.map(({ name, params }) => [name, params.length])).toEqual([
+ ['dynamic', 3], ['dropped', 1],
+ ]);
+ const files = [{ file: 'dynamic.jsx', src: `
+ import { dynamic } from './services/api';
+ dynamic(id, { value: 'literal }, separator' }, { silent: true });
+ dynamic(id, { silent: true });
+ const label = \`prefix \${api.dropped(choose({ nested: [1, 2] }), { silent: true })}\`;
+ // api.dropped(id, { silent: true });
+ const sample = 'api.dropped(id, { silent: true })';
+ ` }];
+ expect(findDroppedSilent(files, wrappers)).toEqual([
+ 'dynamic.jsx:4 dynamic(…) passes { silent } as argument 2',
+ 'dynamic.jsx:5 dropped(…) passes { silent } as argument 2',
+ ]);
+ });
+
it('flags a call whose position the wrapper drops and accepts a forwarding one', () => {
const wrappers = wrapperExports(`
export const noOptions = (id) => request(\`/x/\${id}\`);
diff --git a/server/lib/README.md b/server/lib/README.md
index df63a9a0d9..466567ce71 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -682,7 +682,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `hostControlRoutes.js` | The audited `HOST_CONTROL_ROUTES` list of HTTP routes that execute on the host (app lifecycle, CoS agent queueing, git, scaffold, standardize, feature agents, loops, GSD, local review, provider execution config, autopilot start) plus `hostControlRouteFor` / `isHostControlRoute`; `authGate.js#hostControlRouteGate` applies `requireHostControl` to every entry. `hostControlBodyKeys` names the execution-policy keys of a `PUT /api/settings`, `PATCH /api/settings/slice`, or `PUT /api/cos/config` body (`HOST_CONTROL_SETTINGS_SLICES`, `HOST_CONTROL_OPEN_COS_CONFIG_KEYS`), and `hostControlSettingsPathsIn` / `changedHostControlSettingsPaths` the nested settings keys that pick an executable, an outbound endpoint or a host data file (`HOST_CONTROL_SETTINGS_PATHS`, gated only when the value changes). `voiceConfigHostControlPathsIn` / `changedVoiceConfigHostControlPaths` do the same for the voice agent's instructions, coding-agent switch and STT endpoint (`HOST_CONTROL_VOICE_CONFIG_PATHS` on `PUT /api/voice/config`). All of these are gated by `authGate.js#hostControlBodyGate`. The socket twin is `HOST_CONTROL_SOCKET_EVENTS` in `services/socket.js`. |
| `apiRouteGraph.js` | Static scan of the mounted HTTP route graph — `server/index.js` `app.use('/api/…', router)` mounts → route-module imports → `router.use(prefix, child)` → `router.<method>('<path>')`, including aliased mounts, local subrouters, factory returns and composed `toolkit.routes.<x>` routers. `buildApiRouteCatalog({ repoRoot? })` → `{ derivedFrom, mounts, routes: [{ method, path, mountPath, sources }], stats }`; `getApiRouteCatalog()` derives it on first use and caches it for the process (read by the API Explorer, the internal OpenAPI document and `apiRouteParity.test.js`); `scanRouteGraph()` adds `declarationKeys` / `duplicateDeclarationKeys` (`routeDeclarationKey`) for the coverage guard in `apiRouteGraph.test.js`. Never committed — see "Generated manifests" in `server/AGENTS.md`. |
| `socketEventCatalog.js` | Searchable projection of the cached Socket.IO inventory: direction, domain, and runtime-schema coverage. |
-| `sourceScan.js` | Lexer-assisted primitives shared by the whole-tree source-scan guard suites, so the timer rule and the socket rule cannot drift on what "owns its rejection" means. `blankLiterals(src)` blanks comment/string/template/regex CONTENT to spaces while preserving length, so a brace inside a literal cannot skew a bracket walk and a caller can still read a literal (a socket event name) out of the original string at the same offset; `matchBracket(src, open)` returns the index past the matching `)`/`]`/`}`; `parseCallbackAt(blanked, from, limit)` parses the function expression at `from` (`async`/`function`/either arrow spelling, skipping the parameter list as a unit so a `= {}` default is not mistaken for the body) and returns `{isAsync, start, text}`; `unguardedAwaits(body)` returns the awaited chains that neither sit inside a `try`/`catch` nor END in `.catch(…)`. `blankComments(src)` is the weaker LINE-based stripper the per-line `child_process` rule needs. `blankCommentBodies(src)` is the complement — comments blanked, literal TEXT kept — for a caller that reads a message string and must not see a marker written in a comment beside it. Callers: `childProcess.guards.test.js`, `server/timerCallbackConventions.test.js`, `server/logLevelConventions.test.js`, `server/sockets/asyncHandlerGuard.test.js`, `server/process-safety-net.test.js`. |
+| `sourceScan.js` | Lexer-assisted primitives shared by the whole-tree source-scan guard suites, so the timer rule and the socket rule cannot drift on what "owns its rejection" means. `blankLiterals(src)` blanks comment/string/template/regex CONTENT to spaces while preserving length, so a brace inside a literal cannot skew a bracket walk and a caller can still read a literal (a socket event name) out of the original string at the same offset; `matchBracket(src, open)` returns the index past the matching `)`/`]`/`}`; `parseCallbackAt(blanked, from, limit)` parses the function expression at `from` (`async`/`function`/either arrow spelling, skipping the parameter list as a unit so a `= {}` default is not mistaken for the body) and returns `{isAsync, start, text}`; `unguardedAwaits(body)` returns the awaited chains that neither sit inside a `try`/`catch` nor END in `.catch(…)`. `blankComments(src)` is the weaker LINE-based stripper the per-line `child_process` rule needs. `blankLiterals(src, { includeCommentOnly: true })` returns `{ code, blanked }` (the comment-only and fully blanked views) in one lexer traversal for guards needing both. `blankCommentBodies(src)` is the complement — comments blanked, literal TEXT kept — for a caller that reads a message string and must not see a marker written in a comment beside it. Callers: `childProcess.guards.test.js`, `server/timerCallbackConventions.test.js`, `server/logLevelConventions.test.js`, `server/sockets/asyncHandlerGuard.test.js`, `server/process-safety-net.test.js`. |
| `apiOperationContracts.js` | Detailed operation metadata for intentionally public APIs. It consumes the canonical route Zod contracts and feeds both public and internal OpenAPI documents. |
| `apiRegistry.js` | Single source of truth for which PortOS services are externally-callable HTTP APIs (`voice`, `sdapi`). `API_REGISTRY` declares each API's `publicPrefixes` (read/compute-safe surface only) + defaults; `isRegistryPublic(settings, path)` tells `authGate` when an `exposed && !requireAuth` API re-opens its prefix; `resolveApiAccess(settings)` merges persisted `apiAccess` flags for the Settings UI + OpenAPI docs. |
| `arrayUtils.js` | `asArray(value)` — returns an array unchanged, or a fresh empty array for non-array inputs. `shuffle(arr, random?)` — Fisher-Yates shuffle (new array, never mutates). The canonical uniform shuffle — never `arr.sort(() => Math.random() - 0.5)`, which is biased. Shared by `meatspacePostCognitive.js` (Schulte table / mental rotation) and `meatspacePostMemory.js` (memory drill generators). `dedupeByKey(items, keyOf, pick?)` — one survivor per key, first-seen order. **Required before any multi-row `INSERT … ON CONFLICT (key) DO UPDATE`**: Postgres refuses the whole statement ("ON CONFLICT DO UPDATE command cannot affect row a second time") when its VALUES list names one conflict key twice, and the rows a batcher joins usually come from something that promises no uniqueness (a disk scan, a peer payload). `DO NOTHING` upserts are exempt. `pick(held, candidate)` defaults to last-seen-wins (what a sequential one-row upsert loop leaves); pass a comparator when the table's conflict rule isn't "latest write" — `memorySync.applyRemoteChanges` keeps the newest `updatedAt` so a peer's payload ordering can't flip a last-writer-wins outcome. Used by `services/mediaAssetIndex/db.js` and `services/memorySync.js`. |
diff --git a/server/lib/sourceScan.js b/server/lib/sourceScan.js
index c32757e129..b9e72b676a 100644
--- a/server/lib/sourceScan.js
+++ b/server/lib/sourceScan.js
@@ -89,23 +89,21 @@ function regexCanStartAfter(prev, chars, at) {
return KEYWORDS_BEFORE_REGEX.has(wordEndingAt(chars, end));
}
-/**
- * Replace the contents of comments, string/template literals, and regex
- * literals with spaces, preserving LENGTH so every index still maps back to the
- * original source. Everything downstream (bracket walks, `await` matching) then
- * sees code only — and a caller that needs a literal's text (a socket event
- * name, say) can still read it out of the original string at the same offset.
- * @param {string} src
- * @param {{keepLiteralText?: boolean}} [options] `keepLiteralText` leaves
- * string/template/regex CONTENT in place and blanks only comments — reach it
- * through `blankCommentBodies`. The lex is identical either way, so the two
- * modes cannot disagree about where a comment ends and a literal begins.
- * @returns {string} same length as `src`
- */
-export function blankLiterals(src, { keepLiteralText = false } = {}) {
+// One state machine for both public modes and the paired-view scan. Only the
+// paired view assembles the comment-only output from spans; legacy callers
+// keep one output array.
+function lexSource(src, { keepLiteralText = false, includeCommentOnly = false } = {}) {
const out = src.split('');
+ const comments = includeCommentOnly ? [] : null;
+ let commentEnd = 0;
const n = src.length;
const blank = (i) => { if (i < n && src[i] !== '\n') out[i] = ' '; };
+ const recordComment = (start, end) => {
+ if (!comments) return;
+ const until = Math.min(end, n);
+ comments.push(src.slice(commentEnd, start), out.slice(start, until).join(''));
+ commentEnd = until;
+ };
// Blanks a character belonging to a LITERAL rather than to a comment.
const blankLit = keepLiteralText ? () => {} : blank;
// Brace depths of the code regions opened by `${` inside template literals,
@@ -137,12 +135,16 @@ export function blankLiterals(src, { keepLiteralText = false } = {}) {
}
if (c === '/' && src[i + 1] === '/') {
+ const start = i;
while (i < n && src[i] !== '\n') { blank(i); i += 1; }
+ recordComment(start, i);
continue;
}
if (c === '/' && src[i + 1] === '*') {
+ const start = i;
while (i < n && !(src[i] === '*' && src[i + 1] === '/')) { blank(i); i += 1; }
blank(i); blank(i + 1); i += 2;
+ recordComment(start, i);
continue;
}
if (c === "'" || c === '"') {
@@ -189,7 +191,28 @@ export function blankLiterals(src, { keepLiteralText = false } = {}) {
i += 1;
}
- return out.join('');
+ const blanked = out.join('');
+ return comments ? { blanked, code: comments.join('') + src.slice(commentEnd) } : blanked;
+}
+
+/**
+ * Replace the contents of comments, string/template literals, and regex
+ * literals with spaces, preserving LENGTH so every index still maps back to the
+ * original source. Everything downstream (bracket walks, `await` matching) then
+ * sees code only — and a caller that needs a literal's text (a socket event
+ * name, say) can still read it out of the original string at the same offset.
+ * @param {string} src
+ * @param {{keepLiteralText?: boolean, includeCommentOnly?: boolean}} [options] `keepLiteralText` leaves
+ * string/template/regex CONTENT in place and blanks only comments — reach it
+ * through `blankCommentBodies`. The lex is identical either way, so the two
+ * modes cannot disagree about where a comment ends and a literal begins.
+ * `includeCommentOnly` returns both `{ code, blanked }` views from one
+ * traversal; `code` equals `blankCommentBodies(src)` and `blanked` equals
+ * the default `blankLiterals(src)`. This paired mode overrides `keepLiteralText`.
+ * @returns {string|{code: string, blanked: string}} each view has the same length as `src`
+ */
+export function blankLiterals(src, { keepLiteralText = false, includeCommentOnly = false } = {}) {
+ return lexSource(src, { keepLiteralText: includeCommentOnly ? false : keepLiteralText, includeCommentOnly });
}
/**
diff --git a/server/lib/sourceScan.test.js b/server/lib/sourceScan.test.js
new file mode 100644
index 0000000000..213418a2a8
--- /dev/null
+++ b/server/lib/sourceScan.test.js
@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { blankCommentBodies, blankLiterals } from './sourceScan.js';
+
+describe('single-traversal source views', () => {
+ it('preserves both lexer modes across literal, comment, and interpolation boundaries', () => {
+ const sources = [
+ '// comment with \" and `\ncall("https://example.com/*literal*/", /["\']/g); /* end */',
+ 'const label = `text // ${api.load(id, { silent: true })} ${`nested ${value}`}`;',
+ 'async function read() { return /[)}]/.test(value); } // tail',
+ 'const of = 4; f(of / n); obj.return / count; caféreturn / size;',
+ 'const text = "escaped \\\" quote"; /* multiline\ncomment */ next();',
+ '/* unfinished comment',
+ '// 💡 Unicode and CRLF\r\nconst x = "😀";\n',
+ ];
+ for (const src of sources) {
+ const { code, blanked } = blankLiterals(src, { includeCommentOnly: true });
+ expect(code).toBe(blankCommentBodies(src));
+ expect(blanked).toBe(blankLiterals(src));
+ expect(code.length).toBe(src.length);
+ expect(blanked.length).toBe(src.length);
+ }
+ });
+});