Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Build API-wrapper scan views in one lexer pass (#11561)
[._.]/a983baff59e9
diff --git a/client/src/apiWrapperOptionsConventions.test.js b/client/src/apiWrapperOptionsConventions.test.jsindex d9e4e75a7c..3bf08e1886 100644--- a/client/src/apiWrapperOptionsConventions.test.js+++ b/client/src/apiWrapperOptionsConventions.test.js@@ -42,12 +42,22 @@ import { readFileSync } from 'fs';import { join, dirname } from 'path';import { fileURLToPath } from 'url';import { trackedSourceFiles } from './test/trackedFiles.js';-import { blankCommentBodies, blankLiterals, matchBracket } from '../../server/lib/sourceScan.js';+import { blankLiterals, matchBracket } from '../../server/lib/sourceScan.js';const CLIENT_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');const SERVICE_FILE = /^src\/services\/api[^/]*\.js$/;const OPTIONS_NAMES = new Set(['options', 'opts']);+// Opt-in stage evidence for workload-sensitive timeouts; normal runs stay quiet.+const profile = process.env.PORTOS_PROFILE_API_WRAPPERS === '1' ? new Map() : null;+const measure = (stage, run) => {+ if (!profile) return run();+ const start = performance.now();+ const result = run();+ profile.set(stage, (profile.get(stage) ?? 0) + performance.now() - start);+ return result;+};+/** Split `text[start, end)` on top-level commas, using the blanked copy to find them. */function splitTopLevel(blanked, text, start, end) {const parts = [];@@ -71,8 +81,7 @@ const WRAPPER_HEAD = /^[ \t]*export\s+(?:(?:async\s+)?function\s+(\w+)\s*\(|cons/** Exported function wrappers in one service file: `[{ name, params, text }]`. */export function wrapperExports(src) {- const code = blankCommentBodies(src);- const blanked = blankLiterals(src);+ const { code, blanked } = measure('lex', () => blankLiterals(src, { includeCommentOnly: true }));const heads = [...blanked.matchAll(WRAPPER_HEAD)];const allExports = [...blanked.matchAll(/^[ \t]*export\s/gm)].map((m) => m.index);return heads.map((head) => {@@ -122,8 +131,7 @@ const SILENT_KEY = /[{,]\s*silent\s*(?::|,|\})/;*/export function silentCalls(src, names) {if (!src.includes('silent')) return [];- const code = blankCommentBodies(src);- const blanked = blankLiterals(src);+ const { code, blanked } = measure('lex', () => blankLiterals(src, { includeCommentOnly: true }));const imported = new Set([...code.matchAll(/import\s*\{([^}]*)\}\s*from/g)].flatMap((m) => m[1].split(',').map((part) => part.trim().split(/\s+as\s+/).pop())));const hits = [];@@ -165,13 +173,17 @@ const read = (file) => ({ file, src: readFileSync(join(CLIENT_ROOT, file), 'utf8describe('silent options reach the API wrapper that reads them (#9951)', () => {it('has no { silent } passed to a position the wrapper drops', () => {- const tracked = trackedSourceFiles(CLIENT_ROOT);+ profile?.clear();+ const tracked = measure('discovery', () => trackedSourceFiles(CLIENT_ROOT));expect(tracked.length).toBeGreaterThan(100);- const wrappers = tracked.filter((file) => SERVICE_FILE.test(file)).flatMap((file) => wrapperExports(read(file).src));+ const files = measure('read', () => tracked.map(read));+ const wrappers = measure('wrappers', () => files.filter(({ file }) => SERVICE_FILE.test(file)).flatMap(({ src }) => wrapperExports(src)));// A broken parse would otherwise pass by judging nothing.expect(wrappers.length).toBeGreaterThan(500);- const violations = findDroppedSilent(tracked.map(read), wrappers);+ const violations = measure('calls', () => findDroppedSilent(files, wrappers));+ // Explicit profiling must survive CI's suppression of passing-test consoles.+ if (profile) process.stdout.write(`⏱ API wrapper scan: files=${files.length}, wrappers=${wrappers.length}, ${[...profile].map(([stage, ms]) => `${stage}=${ms.toFixed(1)}ms`).join(', ')}\n`);expect(violations,'These calls pass `{ silent: … }` to an API wrapper whose parameter at that position does not '@@ -219,6 +231,31 @@ describe('silent options reach the API wrapper that reads them (#9951)', () => {expect(silentCalls(`${imp}// noOptions(id, { silent: true })`, names)).toEqual([]);});+ it('keeps dynamic endpoints, nested defaults, and calls inside template interpolation visible', () => {+ const wrappers = wrapperExports(`+ export const dynamic = (id, data = { label: \`item-\${id}\` }, extra) =>+ request(\`/things/\${id ? 'nested)' : 'other'}\`, { body: JSON.stringify(data), ...extra });+ export async function dropped(id = choose({ nested: [1, 2] })) {+ return request(\`/things/\${id}\`);+ }+ `);+ expect(wrappers.map(({ name, params }) => [name, params.length])).toEqual([+ ['dynamic', 3], ['dropped', 1],+ ]);+ const files = [{ file: 'dynamic.jsx', src: `+ import { dynamic } from './services/api';+ dynamic(id, { value: 'literal }, separator' }, { silent: true });+ dynamic(id, { silent: true });+ const label = \`prefix \${api.dropped(choose({ nested: [1, 2] }), { silent: true })}\`;+ // api.dropped(id, { silent: true });+ const sample = 'api.dropped(id, { silent: true })';+ ` }];+ expect(findDroppedSilent(files, wrappers)).toEqual([+ 'dynamic.jsx:4 dynamic(…) passes { silent } as argument 2',+ 'dynamic.jsx:5 dropped(…) passes { silent } as argument 2',+ ]);+ });+it('flags a call whose position the wrapper drops and accepts a forwarding one', () => {const wrappers = wrapperExports(`export const noOptions = (id) => request(\`/x/\${id}\`);diff --git a/server/lib/README.md b/server/lib/README.mdindex df63a9a0d9..466567ce71 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -682,7 +682,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `hostControlRoutes.js` | The audited `HOST_CONTROL_ROUTES` list of HTTP routes that execute on the host (app lifecycle, CoS agent queueing, git, scaffold, standardize, feature agents, loops, GSD, local review, provider execution config, autopilot start) plus `hostControlRouteFor` / `isHostControlRoute`; `authGate.js#hostControlRouteGate` applies `requireHostControl` to every entry. `hostControlBodyKeys` names the execution-policy keys of a `PUT /api/settings`, `PATCH /api/settings/slice`, or `PUT /api/cos/config` body (`HOST_CONTROL_SETTINGS_SLICES`, `HOST_CONTROL_OPEN_COS_CONFIG_KEYS`), and `hostControlSettingsPathsIn` / `changedHostControlSettingsPaths` the nested settings keys that pick an executable, an outbound endpoint or a host data file (`HOST_CONTROL_SETTINGS_PATHS`, gated only when the value changes). `voiceConfigHostControlPathsIn` / `changedVoiceConfigHostControlPaths` do the same for the voice agent's instructions, coding-agent switch and STT endpoint (`HOST_CONTROL_VOICE_CONFIG_PATHS` on `PUT /api/voice/config`). All of these are gated by `authGate.js#hostControlBodyGate`. The socket twin is `HOST_CONTROL_SOCKET_EVENTS` in `services/socket.js`. || `apiRouteGraph.js` | Static scan of the mounted HTTP route graph — `server/index.js` `app.use('/api/…', router)` mounts → route-module imports → `router.use(prefix, child)` → `router.<method>('<path>')`, including aliased mounts, local subrouters, factory returns and composed `toolkit.routes.<x>` routers. `buildApiRouteCatalog({ repoRoot? })` → `{ derivedFrom, mounts, routes: [{ method, path, mountPath, sources }], stats }`; `getApiRouteCatalog()` derives it on first use and caches it for the process (read by the API Explorer, the internal OpenAPI document and `apiRouteParity.test.js`); `scanRouteGraph()` adds `declarationKeys` / `duplicateDeclarationKeys` (`routeDeclarationKey`) for the coverage guard in `apiRouteGraph.test.js`. Never committed — see "Generated manifests" in `server/AGENTS.md`. || `socketEventCatalog.js` | Searchable projection of the cached Socket.IO inventory: direction, domain, and runtime-schema coverage. |-| `sourceScan.js` | Lexer-assisted primitives shared by the whole-tree source-scan guard suites, so the timer rule and the socket rule cannot drift on what "owns its rejection" means. `blankLiterals(src)` blanks comment/string/template/regex CONTENT to spaces while preserving length, so a brace inside a literal cannot skew a bracket walk and a caller can still read a literal (a socket event name) out of the original string at the same offset; `matchBracket(src, open)` returns the index past the matching `)`/`]`/`}`; `parseCallbackAt(blanked, from, limit)` parses the function expression at `from` (`async`/`function`/either arrow spelling, skipping the parameter list as a unit so a `= {}` default is not mistaken for the body) and returns `{isAsync, start, text}`; `unguardedAwaits(body)` returns the awaited chains that neither sit inside a `try`/`catch` nor END in `.catch(…)`. `blankComments(src)` is the weaker LINE-based stripper the per-line `child_process` rule needs. `blankCommentBodies(src)` is the complement — comments blanked, literal TEXT kept — for a caller that reads a message string and must not see a marker written in a comment beside it. Callers: `childProcess.guards.test.js`, `server/timerCallbackConventions.test.js`, `server/logLevelConventions.test.js`, `server/sockets/asyncHandlerGuard.test.js`, `server/process-safety-net.test.js`. |+| `sourceScan.js` | Lexer-assisted primitives shared by the whole-tree source-scan guard suites, so the timer rule and the socket rule cannot drift on what "owns its rejection" means. `blankLiterals(src)` blanks comment/string/template/regex CONTENT to spaces while preserving length, so a brace inside a literal cannot skew a bracket walk and a caller can still read a literal (a socket event name) out of the original string at the same offset; `matchBracket(src, open)` returns the index past the matching `)`/`]`/`}`; `parseCallbackAt(blanked, from, limit)` parses the function expression at `from` (`async`/`function`/either arrow spelling, skipping the parameter list as a unit so a `= {}` default is not mistaken for the body) and returns `{isAsync, start, text}`; `unguardedAwaits(body)` returns the awaited chains that neither sit inside a `try`/`catch` nor END in `.catch(…)`. `blankComments(src)` is the weaker LINE-based stripper the per-line `child_process` rule needs. `blankLiterals(src, { includeCommentOnly: true })` returns `{ code, blanked }` (the comment-only and fully blanked views) in one lexer traversal for guards needing both. `blankCommentBodies(src)` is the complement — comments blanked, literal TEXT kept — for a caller that reads a message string and must not see a marker written in a comment beside it. Callers: `childProcess.guards.test.js`, `server/timerCallbackConventions.test.js`, `server/logLevelConventions.test.js`, `server/sockets/asyncHandlerGuard.test.js`, `server/process-safety-net.test.js`. || `apiOperationContracts.js` | Detailed operation metadata for intentionally public APIs. It consumes the canonical route Zod contracts and feeds both public and internal OpenAPI documents. || `apiRegistry.js` | Single source of truth for which PortOS services are externally-callable HTTP APIs (`voice`, `sdapi`). `API_REGISTRY` declares each API's `publicPrefixes` (read/compute-safe surface only) + defaults; `isRegistryPublic(settings, path)` tells `authGate` when an `exposed && !requireAuth` API re-opens its prefix; `resolveApiAccess(settings)` merges persisted `apiAccess` flags for the Settings UI + OpenAPI docs. || `arrayUtils.js` | `asArray(value)` — returns an array unchanged, or a fresh empty array for non-array inputs. `shuffle(arr, random?)` — Fisher-Yates shuffle (new array, never mutates). The canonical uniform shuffle — never `arr.sort(() => Math.random() - 0.5)`, which is biased. Shared by `meatspacePostCognitive.js` (Schulte table / mental rotation) and `meatspacePostMemory.js` (memory drill generators). `dedupeByKey(items, keyOf, pick?)` — one survivor per key, first-seen order. **Required before any multi-row `INSERT … ON CONFLICT (key) DO UPDATE`**: Postgres refuses the whole statement ("ON CONFLICT DO UPDATE command cannot affect row a second time") when its VALUES list names one conflict key twice, and the rows a batcher joins usually come from something that promises no uniqueness (a disk scan, a peer payload). `DO NOTHING` upserts are exempt. `pick(held, candidate)` defaults to last-seen-wins (what a sequential one-row upsert loop leaves); pass a comparator when the table's conflict rule isn't "latest write" — `memorySync.applyRemoteChanges` keeps the newest `updatedAt` so a peer's payload ordering can't flip a last-writer-wins outcome. Used by `services/mediaAssetIndex/db.js` and `services/memorySync.js`. |diff --git a/server/lib/sourceScan.js b/server/lib/sourceScan.jsindex c32757e129..b9e72b676a 100644--- a/server/lib/sourceScan.js+++ b/server/lib/sourceScan.js@@ -89,23 +89,21 @@ function regexCanStartAfter(prev, chars, at) {return KEYWORDS_BEFORE_REGEX.has(wordEndingAt(chars, end));}-/**- * Replace the contents of comments, string/template literals, and regex- * literals with spaces, preserving LENGTH so every index still maps back to the- * original source. Everything downstream (bracket walks, `await` matching) then- * sees code only — and a caller that needs a literal's text (a socket event- * name, say) can still read it out of the original string at the same offset.- * @param {string} src- * @param {{keepLiteralText?: boolean}} [options] `keepLiteralText` leaves- * string/template/regex CONTENT in place and blanks only comments — reach it- * through `blankCommentBodies`. The lex is identical either way, so the two- * modes cannot disagree about where a comment ends and a literal begins.- * @returns {string} same length as `src`- */-export function blankLiterals(src, { keepLiteralText = false } = {}) {+// One state machine for both public modes and the paired-view scan. Only the+// paired view assembles the comment-only output from spans; legacy callers+// keep one output array.+function lexSource(src, { keepLiteralText = false, includeCommentOnly = false } = {}) {const out = src.split('');+ const comments = includeCommentOnly ? [] : null;+ let commentEnd = 0;const n = src.length;const blank = (i) => { if (i < n && src[i] !== '\n') out[i] = ' '; };+ const recordComment = (start, end) => {+ if (!comments) return;+ const until = Math.min(end, n);+ comments.push(src.slice(commentEnd, start), out.slice(start, until).join(''));+ commentEnd = until;+ };// Blanks a character belonging to a LITERAL rather than to a comment.const blankLit = keepLiteralText ? () => {} : blank;// Brace depths of the code regions opened by `${` inside template literals,@@ -137,12 +135,16 @@ export function blankLiterals(src, { keepLiteralText = false } = {}) {}if (c === '/' && src[i + 1] === '/') {+ const start = i;while (i < n && src[i] !== '\n') { blank(i); i += 1; }+ recordComment(start, i);continue;}if (c === '/' && src[i + 1] === '*') {+ const start = i;while (i < n && !(src[i] === '*' && src[i + 1] === '/')) { blank(i); i += 1; }blank(i); blank(i + 1); i += 2;+ recordComment(start, i);continue;}if (c === "'" || c === '"') {@@ -189,7 +191,28 @@ export function blankLiterals(src, { keepLiteralText = false } = {}) {i += 1;}- return out.join('');+ const blanked = out.join('');+ return comments ? { blanked, code: comments.join('') + src.slice(commentEnd) } : blanked;+}++/**+ * Replace the contents of comments, string/template literals, and regex+ * literals with spaces, preserving LENGTH so every index still maps back to the+ * original source. Everything downstream (bracket walks, `await` matching) then+ * sees code only — and a caller that needs a literal's text (a socket event+ * name, say) can still read it out of the original string at the same offset.+ * @param {string} src+ * @param {{keepLiteralText?: boolean, includeCommentOnly?: boolean}} [options] `keepLiteralText` leaves+ * string/template/regex CONTENT in place and blanks only comments — reach it+ * through `blankCommentBodies`. The lex is identical either way, so the two+ * modes cannot disagree about where a comment ends and a literal begins.+ * `includeCommentOnly` returns both `{ code, blanked }` views from one+ * traversal; `code` equals `blankCommentBodies(src)` and `blanked` equals+ * the default `blankLiterals(src)`. This paired mode overrides `keepLiteralText`.+ * @returns {string|{code: string, blanked: string}} each view has the same length as `src`+ */+export function blankLiterals(src, { keepLiteralText = false, includeCommentOnly = false } = {}) {+ return lexSource(src, { keepLiteralText: includeCommentOnly ? false : keepLiteralText, includeCommentOnly });}/**diff --git a/server/lib/sourceScan.test.js b/server/lib/sourceScan.test.jsnew file mode 100644index 0000000000..213418a2a8--- /dev/null+++ b/server/lib/sourceScan.test.js@@ -0,0 +1,23 @@+import { describe, expect, it } from 'vitest';+import { blankCommentBodies, blankLiterals } from './sourceScan.js';++describe('single-traversal source views', () => {+ it('preserves both lexer modes across literal, comment, and interpolation boundaries', () => {+ const sources = [+ '// comment with \" and `\ncall("https://example.com/*literal*/", /["\']/g); /* end */',+ 'const label = `text // ${api.load(id, { silent: true })} ${`nested ${value}`}`;',+ 'async function read() { return /[)}]/.test(value); } // tail',+ 'const of = 4; f(of / n); obj.return / count; caféreturn / size;',+ 'const text = "escaped \\\" quote"; /* multiline\ncomment */ next();',+ '/* unfinished comment',+ '// 💡 Unicode and CRLF\r\nconst x = "😀";\n',+ ];+ for (const src of sources) {+ const { code, blanked } = blankLiterals(src, { includeCommentOnly: true });+ expect(code).toBe(blankCommentBodies(src));+ expect(blanked).toBe(blankLiterals(src));+ expect(code.length).toBe(src.length);+ expect(blanked.length).toBe(src.length);+ }+ });+});