Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
refactor(ok): Wave 1 capability service extraction (PRD-7787) (#3221)
omar-inkeep998e1493da5e
diff --git a/knip.config.ts b/knip.config.tsindex f004e6bc3..13582ee18 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -123,9 +123,7 @@ export default {'packages/server': {entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],project: 'src/**',- ignoreDependencies: [- '@types/shell-quote',- ],+ ignoreDependencies: ['@types/shell-quote'],},'packages/cli': {entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],diff --git a/packages/app/tests/meta/getfileindex-allfiles-coverage.test.ts b/packages/app/tests/meta/getfileindex-allfiles-coverage.test.tsindex 0ff14a0a2..386f14218 100644--- a/packages/app/tests/meta/getfileindex-allfiles-coverage.test.ts+++ b/packages/app/tests/meta/getfileindex-allfiles-coverage.test.ts@@ -32,6 +32,12 @@ const FILE_WATCHER_PATH = join(SERVER_SRC_ROOT, 'file-watcher.ts');// hands the accessor closure to api-extension; it never iterates file entries// itself, so it cannot leak a `kind:'file'` entry to a markdown-assuming path.const SERVER_FACTORY_PATH = join(SERVER_SRC_ROOT, 'server-factory.ts');+// The extracted search service: hosts the corpus-build sites+// (`buildWorkspaceSearchDocumentsFromIndex` / `workspaceSearchFingerprint` /+// `deriveFolderSearchDocuments`) that previously lived in api-extension.ts.+// api-extension wires the accessor into `createSearchService({ getAllFilesIndex })`+// the same way server-factory wires it into createApiExtension.+const SEARCH_SERVICE_PATH = join(SERVER_SRC_ROOT, 'services/search.ts');/*** Function-name allowlist for `getAllFilesIndex()` (or `includeFiles:true`)@@ -151,7 +157,10 @@ function listProductionTsFiles(dir: string): string[] {describe('PRD-7117 US-002 — getAllFilesIndex caller coverage (D12 §13-A)', () => {test('every getAllFilesIndex() call site in api-extension.ts is allowlisted or kind-filtered', () => {- const sites = collectAllFilesCallSites(API_EXT_PATH);+ const sites = [+ ...collectAllFilesCallSites(API_EXT_PATH),+ ...collectAllFilesCallSites(SEARCH_SERVICE_PATH),+ ];const failures: string[] = [];for (const site of sites) {const allowed = ALLOWLISTED_SITES.has(site.fn);@@ -184,7 +193,12 @@ describe('PRD-7117 US-002 — getAllFilesIndex caller coverage (D12 §13-A)', ()// doc comment in app's file-tree-utils). A cross-layer leak is therefore// structurally impossible without first adding a server export, which this// server-scoped scan would catch at that export site.- const allowedFiles = new Set([FILE_WATCHER_PATH, API_EXT_PATH, SERVER_FACTORY_PATH]);+ const allowedFiles = new Set([+ FILE_WATCHER_PATH,+ API_EXT_PATH,+ SERVER_FACTORY_PATH,+ SEARCH_SERVICE_PATH,+ ]);const offenders: string[] = [];for (const file of listProductionTsFiles(SERVER_SRC_ROOT)) {if (allowedFiles.has(file)) continue;@@ -199,7 +213,7 @@ describe('PRD-7117 US-002 — getAllFilesIndex caller coverage (D12 §13-A)', ()test('ALLOWLISTED_SITES function names actually exist in api-extension.ts', () => {// Guard against allowlist rot: if a site is renamed or removed without// updating ALLOWLISTED_SITES, the entry becomes a dead authorization.- const source = readFileSync(API_EXT_PATH, 'utf8');+ const source = readFileSync(API_EXT_PATH, 'utf8') + readFileSync(SEARCH_SERVICE_PATH, 'utf8');const missing: string[] = [];for (const name of ALLOWLISTED_SITES) {const fnRe = new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\(`);diff --git a/packages/core/src/bridge/bind-frontmatter-doc.ts b/packages/core/src/bridge/bind-frontmatter-doc.tsindex dfd95ef49..d7a845280 100644--- a/packages/core/src/bridge/bind-frontmatter-doc.ts+++ b/packages/core/src/bridge/bind-frontmatter-doc.ts@@ -1,4 +1,3 @@-import type * as Y from 'yjs';import type { Err, Ok, Result } from '../config/result.ts';import { type FrontmatterValidationError, toFrontmatterIssue } from '../frontmatter/errors.ts';diff --git a/packages/core/src/bridge/bridge-invariant.ts b/packages/core/src/bridge/bridge-invariant.tsindex 8bfdb40cd..005627195 100644--- a/packages/core/src/bridge/bridge-invariant.ts+++ b/packages/core/src/bridge/bridge-invariant.ts@@ -1,4 +1,3 @@-import { fnv1aDigest } from './hash-util.ts';export type BridgeInvariantSite = 'observer-b' | 'persistence' | 'test-harness';diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsindex 2a597ec3d..f4cc8a286 100644--- a/packages/core/src/bridge/compose.ts+++ b/packages/core/src/bridge/compose.ts@@ -1,4 +1,3 @@-import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.tsindex 01ef76a9f..8fb428b90 100644--- a/packages/core/src/bridge/doc-boundary-space.ts+++ b/packages/core/src/bridge/doc-boundary-space.ts@@ -1,4 +1,3 @@-import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';diff --git a/packages/core/src/bridge/growth-detect.ts b/packages/core/src/bridge/growth-detect.tsindex 463e89595..9d9c244b0 100644--- a/packages/core/src/bridge/growth-detect.ts+++ b/packages/core/src/bridge/growth-detect.ts@@ -1,4 +1,3 @@-const DEFAULT_MIN_SUBSTANTIVE_LINE_LENGTH = 16;export const DUPLICATION_GATE_MIN_LINE_LENGTH = 8;diff --git a/packages/core/src/bridge/intraline-predicate.ts b/packages/core/src/bridge/intraline-predicate.tsindex 0939885bb..c43fdd7e7 100644--- a/packages/core/src/bridge/intraline-predicate.ts+++ b/packages/core/src/bridge/intraline-predicate.ts@@ -1,4 +1,3 @@-export function rawLineCounts(doc: string): Map<string, number> {const counts = new Map<string, number>();for (const line of doc.split('\n')) {diff --git a/packages/core/src/bridge/merge-three-way.ts b/packages/core/src/bridge/merge-three-way.tsindex c0dc65705..251560a35 100644--- a/packages/core/src/bridge/merge-three-way.ts+++ b/packages/core/src/bridge/merge-three-way.ts@@ -65,7 +65,6 @@ export function tryLineLevelCombine(return { clean: true, merged };}-export type BridgeMergeContentLossSide = 'user' | 'agent';export type BridgeMergeContentLossWhich = 'substring' | 'order' | 'growth';diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.tsindex dcc5b0690..01ac80099 100644--- a/packages/core/src/bridge/normalize.ts+++ b/packages/core/src/bridge/normalize.ts@@ -1,4 +1,3 @@-import { stripFrontmatter } from '../extensions/frontmatter.ts';import { bodyEdgeEmpties } from './doc-boundary-space.ts';diff --git a/packages/core/src/bridge/pm-structural-equivalence.ts b/packages/core/src/bridge/pm-structural-equivalence.tsindex 0be5203e4..754dc5cc2 100644--- a/packages/core/src/bridge/pm-structural-equivalence.ts+++ b/packages/core/src/bridge/pm-structural-equivalence.ts@@ -32,7 +32,6 @@ const BR_SENTINEL: PmStructuralNode = { type: '__br__' };const BR_LITERAL_RE = /^<br\s*\/?>$/i;-interface DegradeEntry {readonly label: string;readonly severity: ToleranceClassSeverity;@@ -128,7 +127,6 @@ export interface ComparePmStructuralOptions {ignoreAttrs?: (attrKey: string) => boolean;}-/** Rebuild a node with `fn` applied to each child; identity when childless.* Shared by the degrade canonicalizers so none re-implements the walk. */function mapChildren(@@ -250,7 +248,6 @@ function applyDegrades(tree: PmStructuralNode): {return { normalized: current, fired };}-export function comparePmStructural(expected: PmStructuralNode,actual: PmStructuralNode,diff --git a/packages/core/src/bridge/subsequence.ts b/packages/core/src/bridge/subsequence.tsindex dca644141..af20de1e2 100644--- a/packages/core/src/bridge/subsequence.ts+++ b/packages/core/src/bridge/subsequence.ts@@ -1,4 +1,3 @@-/** True when `needle` is a subsequence of `haystack` (two-pointer, O(n+m)):* insertions in `haystack` are free; any dropped or substituted `needle` byte* fails. */diff --git a/packages/core/src/markdown/callout-transformer.ts b/packages/core/src/markdown/callout-transformer.tsindex a363dfd54..83d47404a 100644--- a/packages/core/src/markdown/callout-transformer.ts+++ b/packages/core/src/markdown/callout-transformer.ts@@ -1,4 +1,3 @@-import type { Blockquote, Paragraph, Root } from 'mdast';import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';import { visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/code-fence.ts b/packages/core/src/markdown/code-fence.tsindex 4499dbc92..9807a7e82 100644--- a/packages/core/src/markdown/code-fence.ts+++ b/packages/core/src/markdown/code-fence.ts@@ -1,4 +1,3 @@-export function selectFenceChar(info: string): '`' | '~' {return info.includes('`') ? '~' : '`';}diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.tsindex 34704165b..7415c474a 100644--- a/packages/core/src/markdown/comment-promoter.ts+++ b/packages/core/src/markdown/comment-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/dedent-block-jsx-close.ts b/packages/core/src/markdown/dedent-block-jsx-close.tsindex b6299a948..ece5c02a1 100644--- a/packages/core/src/markdown/dedent-block-jsx-close.ts+++ b/packages/core/src/markdown/dedent-block-jsx-close.ts@@ -1,4 +1,3 @@-import { findFencedRegions, isInsideFence } from './fence-regions.ts';const INDENTED_BLOCK_JSX_CLOSE_RE = /^([ ]{1,3})(<\/[A-Z][A-Za-z0-9_]*\s*>)([ \t]*)$/gm;diff --git a/packages/core/src/markdown/details-accordion-promoter.ts b/packages/core/src/markdown/details-accordion-promoter.tsindex acd398aa6..54774764a 100644--- a/packages/core/src/markdown/details-accordion-promoter.ts+++ b/packages/core/src/markdown/details-accordion-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast';import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';import { visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/div-align-promoter.ts b/packages/core/src/markdown/div-align-promoter.tsindex ba9b56384..43ecdc709 100644--- a/packages/core/src/markdown/div-align-promoter.ts+++ b/packages/core/src/markdown/div-align-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast';import type { MdxJsxFlowElement } from 'mdast-util-mdx';import { visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.tsindex 4513da9bb..6d3f3643e 100644--- a/packages/core/src/markdown/fence-regions.ts+++ b/packages/core/src/markdown/fence-regions.ts@@ -1,4 +1,3 @@-const FENCE_RE = /^(`{3,}|~{3,})/gm;export function findFencedRegions(src: string): Array<[number, number]> {diff --git a/packages/core/src/markdown/fixtures/index.ts b/packages/core/src/markdown/fixtures/index.tsindex 6931623c7..00f5771ee 100644--- a/packages/core/src/markdown/fixtures/index.ts+++ b/packages/core/src/markdown/fixtures/index.ts@@ -1,4 +1,3 @@-import { readFileSync } from 'node:fs';import { dirname, resolve } from 'node:path';import { fileURLToPath } from 'node:url';@@ -9,7 +8,6 @@ function fixturePath(...segments: string[]): string {return resolve(FIXTURES_DIR, ...segments);}-interface GfmExample {section: string;markdown: string;@@ -19,7 +17,6 @@ export function loadGfmExamples(): GfmExample[] {return JSON.parse(readFileSync(fixturePath('gfm', 'examples.json'), 'utf8')) as GfmExample[];}-interface MdxCrashEntry {id: string;input: string;@@ -71,7 +68,6 @@ export function loadLargeEmbedFixtures(): LargeEmbedFixture[] {) as LargeEmbedFixture[];}-export function loadPrd6955Before(): string {return readFileSync(fixturePath('regression', 'prd-6955-before.md'), 'utf8');}@@ -80,7 +76,6 @@ export function loadPrd6955CorruptedTriplicated(): string {return readFileSync(fixturePath('regression', 'prd-6955-corrupted-triplicated.md'), 'utf8');}-export interface NgPinnedCase {id: string;name: string;@@ -97,7 +92,6 @@ export function loadNgPinnedCases(): NgPinnedCase[] {) as NgPinnedCase[];}-export function loadLargeRealistic(): string {return readFileSync(fixturePath('perf', 'large-realistic.md'), 'utf8');}diff --git a/packages/core/src/markdown/fixtures/perf/generate.ts b/packages/core/src/markdown/fixtures/perf/generate.tsindex daa3ef82b..8b1eb8519 100644--- a/packages/core/src/markdown/fixtures/perf/generate.ts+++ b/packages/core/src/markdown/fixtures/perf/generate.ts@@ -1,4 +1,3 @@-import { writeFileSync } from 'node:fs';import { dirname, resolve } from 'node:path';import { fileURLToPath } from 'node:url';@@ -11,7 +10,6 @@ const BASELINE_ONLY_COUNTS = [500, 2500] as const;const SEED = 0xf1de1117;-function mulberry32(seed: number): () => number {let a = seed >>> 0;return () => {@@ -38,7 +36,6 @@ function pickWeighted<T>(rand: () => number, items: readonly [T, number][]): T {return items[items.length - 1][0];}-const WORDS = ['lorem','ipsum',@@ -156,7 +153,6 @@ function mdxBlock(rand: () => number): string {return `<${name}>\n\n${sentence(rand)}\n\n</${name}>`;}-type BlockKind = 'paragraph' | 'heading' | 'list' | 'code' | 'table' | 'mdx';const MIX: readonly [BlockKind, number][] = [@@ -195,7 +191,6 @@ function generateDocument(blockCount: number, seed: number): string {return `${blocks.join('\n\n')}\n`;}-function main(): void {for (const count of [...BLOCK_COUNTS, ...BASELINE_ONLY_COUNTS]) {const doc = generateDocument(count, SEED);diff --git a/packages/core/src/markdown/guard-flanking-matrix.ts b/packages/core/src/markdown/guard-flanking-matrix.tsindex 32c055c5b..420dc0fe5 100644--- a/packages/core/src/markdown/guard-flanking-matrix.ts+++ b/packages/core/src/markdown/guard-flanking-matrix.ts@@ -7,7 +7,6 @@ import {import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts';import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts';-export const ATTENTION_DELIMITERS = ['*', '_', '**', '~~', '=='] as const;export type FlankClass = 'whitespace' | 'punctuation' | 'other';diff --git a/packages/core/src/markdown/handler-shadow-audit.ts b/packages/core/src/markdown/handler-shadow-audit.tsindex 283913255..2653473da 100644--- a/packages/core/src/markdown/handler-shadow-audit.ts+++ b/packages/core/src/markdown/handler-shadow-audit.ts@@ -1,4 +1,3 @@-export interface HandlerShadowWitness {input: string;expect: 'byte' | 'byte-and-reparse-type';diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.tsindex 5d1000cc4..da0a81d23 100644--- a/packages/core/src/markdown/highlight-promoter.ts+++ b/packages/core/src/markdown/highlight-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Parent, PhrasingContent, Root, Text } from 'mdast';import type { MdxJsxTextElement } from 'mdast-util-mdx';import { SKIP, visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.tsindex 9681c0056..787315157 100644--- a/packages/core/src/markdown/html-to-mdast.ts+++ b/packages/core/src/markdown/html-to-mdast.ts@@ -1,4 +1,3 @@-import type { Element as HastElement, Root as HastRoot } from 'hast';import type { List, Nodes as MdastNodes, Root as MdastRoot } from 'mdast';import rehypeParse from 'rehype-parse';@@ -129,8 +128,7 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRothrow new HtmlPayloadTooLargeError(html.length, maxBytes);}- const processor = unified()- .use(rehypeParse, { fragment: true });+ const processor = unified().use(rehypeParse, { fragment: true });for (const plugin of cleanupPlugins) {processor.use(plugin);diff --git a/packages/core/src/markdown/image-promoter.ts b/packages/core/src/markdown/image-promoter.tsindex 994eb6917..706fa9afb 100644--- a/packages/core/src/markdown/image-promoter.ts+++ b/packages/core/src/markdown/image-promoter.ts@@ -1,4 +1,3 @@-import type { Image, Paragraph, Root } from 'mdast';import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';import { visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.tsindex 69221627d..9075aa7f1 100644--- a/packages/core/src/markdown/index.ts+++ b/packages/core/src/markdown/index.ts@@ -1,4 +1,3 @@-import {type FromProseMirrorOptions,fromPmMark,@@ -207,7 +206,6 @@ export class MarkdownManager {}}-const registry = createRegistry();function destructureAttrs(@@ -256,7 +254,6 @@ function destructureAttrs(return result;}-function hasDirtyDescendant(node: PmNode): boolean {let found = false;node.descendants((child) => {@@ -279,7 +276,6 @@ function effectiveDirty(node: PmNode, freshnessChecker?: StructuralFreshnessChecreturn freshnessChecker ? freshnessChecker.isDiverged(node.toJSON()) : false;}-function isEmptyMdastParagraph(node: MdastNodes): boolean {if (node.type !== 'paragraph') return false;const children = node.children ?? [];@@ -348,7 +344,6 @@ function extractTextFromMdastNodes(nodes: MdastNodes[]): string {return out;}-import {AUDIO_EXTENSIONS,FILE_ATTACHMENT_EXTENSIONS,@@ -601,7 +596,6 @@ function buildMdastToPmHandlers(}));}-if (m.emphasis) {handlers.emphasis = toPmMark(m.emphasis, (node: Emphasis) => ({sourceDelimiter: node.data?.sourceDelimiter ?? '*',@@ -682,7 +676,6 @@ function buildMdastToPmHandlers(}));}-if (m.link) {const sourceLiteralMark = m.sourceLiteral;handlers.link = (node: Link, _parent: MdastParent, state: MdastToPmState) => {@@ -980,7 +973,6 @@ function buildMdastToPmHandlers(};}-const blockUnknownHandler = (node: {type: string;position?: { start: { offset: number }; end: { offset: number } };@@ -1106,7 +1098,6 @@ function buildMdastToPmHandlers(return handlers as RemarkProseMirrorOptions['handlers'];}-function buildPmToMdastHandlers(schema: Schema,freshness: FreshnessCheckerHolder,diff --git a/packages/core/src/markdown/lint/applies-to.ts b/packages/core/src/markdown/lint/applies-to.tsindex b79e9aef7..fa0be9e86 100644--- a/packages/core/src/markdown/lint/applies-to.ts+++ b/packages/core/src/markdown/lint/applies-to.ts@@ -1,4 +1,3 @@-import picomatch from 'picomatch';import { SUPPORTED_DOC_EXTENSIONS } from '../../constants/doc-extensions.ts';diff --git a/packages/core/src/markdown/lint/config-files.ts b/packages/core/src/markdown/lint/config-files.tsindex 94b34dee1..312c9c74b 100644--- a/packages/core/src/markdown/lint/config-files.ts+++ b/packages/core/src/markdown/lint/config-files.ts@@ -1,4 +1,3 @@-const MARKDOWNLINT_JSON_CONFIG_FILES: ReadonlySet<string> = new Set(['.markdownlint.json','.markdownlint.jsonc',diff --git a/packages/core/src/markdown/lint/config-schemas.ts b/packages/core/src/markdown/lint/config-schemas.tsindex e413367ef..0ba33b7d8 100644--- a/packages/core/src/markdown/lint/config-schemas.ts+++ b/packages/core/src/markdown/lint/config-schemas.ts@@ -38,7 +38,6 @@ const fullPluginShape = Object.fromEntries(LINT_PLUGINS.map((plugin) => [plugin.id, plugin.sliceSchema]),) as z.ZodRawShape;-export const LinterConfigSchema = z.object({enabled: z.boolean(),plugins: z.object(fullPluginShape),diff --git a/packages/core/src/markdown/lint/default-config.ts b/packages/core/src/markdown/lint/default-config.tsindex a51f2ffb0..d48eea5e4 100644--- a/packages/core/src/markdown/lint/default-config.ts+++ b/packages/core/src/markdown/lint/default-config.ts@@ -6,7 +6,6 @@ export const DEFAULT_MARKDOWNLINT_CONFIG: Record<string, MarkdownlintRuleSettingMD013: false,};-export function resolveMarkdownlintConfig(rules: Record<string, MarkdownlintRuleSetting> | undefined,): Configuration {diff --git a/packages/core/src/markdown/lint/frontmatter-schema-edit.ts b/packages/core/src/markdown/lint/frontmatter-schema-edit.tsindex 9c9542661..ef478e268 100644--- a/packages/core/src/markdown/lint/frontmatter-schema-edit.ts+++ b/packages/core/src/markdown/lint/frontmatter-schema-edit.ts@@ -1,4 +1,3 @@-import { CANONICAL_SCHEMA_DIALECT_URIS, DEFAULT_SCHEMA_DIALECT } from './frontmatter-validate.ts';export interface FrontmatterFieldConstraint {diff --git a/packages/core/src/markdown/lint/frontmatter-validate.ts b/packages/core/src/markdown/lint/frontmatter-validate.tsindex b96d19182..0d4c9a0df 100644--- a/packages/core/src/markdown/lint/frontmatter-validate.ts+++ b/packages/core/src/markdown/lint/frontmatter-validate.ts@@ -1,4 +1,3 @@-import Ajv, { type ValidateFunction } from 'ajv';import Ajv2019 from 'ajv/dist/2019.js';import Ajv2020 from 'ajv/dist/2020.js';diff --git a/packages/core/src/markdown/lint/index.ts b/packages/core/src/markdown/lint/index.tsindex 48d260ed6..00d08265a 100644--- a/packages/core/src/markdown/lint/index.ts+++ b/packages/core/src/markdown/lint/index.ts@@ -1,4 +1,3 @@-import { LINT_PLUGINS, type LinterConfig } from './plugins.ts';import type { LintDiagnostic } from './types.ts';diff --git a/packages/core/src/markdown/lint/plugins.ts b/packages/core/src/markdown/lint/plugins.tsindex be5871ed9..5e832aa19 100644--- a/packages/core/src/markdown/lint/plugins.ts+++ b/packages/core/src/markdown/lint/plugins.ts@@ -1,4 +1,3 @@-import { z } from 'zod';import { DEFAULT_MARKDOWNLINT_CONFIG, resolveMarkdownlintConfig } from './default-config.ts';import {@@ -14,7 +13,6 @@ import {type MarkdownlintSlice,} from './types.ts';-const MarkdownlintRuleSettingSchema = z.union([z.boolean(),z.enum(MARKDOWNLINT_RULE_SEVERITIES),diff --git a/packages/core/src/markdown/lint/types.ts b/packages/core/src/markdown/lint/types.tsindex 833c3a727..3918e657c 100644--- a/packages/core/src/markdown/lint/types.ts+++ b/packages/core/src/markdown/lint/types.ts@@ -1,4 +1,3 @@-export const LINT_PLUGIN_IDS = ['markdownlint', 'frontmatter'] as const;export type LintPluginId = (typeof LINT_PLUGIN_IDS)[number];@@ -72,7 +71,6 @@ export interface FrontmatterSlice {schemas: ResolvedFrontmatterSchemaEntry[];}-interface RuleOptionSpecBase {key: string;description: string;diff --git a/packages/core/src/markdown/lint/validation-counts.ts b/packages/core/src/markdown/lint/validation-counts.tsindex 437cc539a..98238656e 100644--- a/packages/core/src/markdown/lint/validation-counts.ts+++ b/packages/core/src/markdown/lint/validation-counts.ts@@ -1,4 +1,3 @@-export interface ValidationSourceCounts {errorCount: number;warningCount: number;diff --git a/packages/core/src/markdown/math-promoter.ts b/packages/core/src/markdown/math-promoter.tsindex bd79424c7..a5b511f25 100644--- a/packages/core/src/markdown/math-promoter.ts+++ b/packages/core/src/markdown/math-promoter.ts@@ -1,4 +1,3 @@-import type { Code, Root } from 'mdast';import type { Math as MdastMath } from 'mdast-util-math';import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.tsindex 00fbc0035..ed89371d0 100644--- a/packages/core/src/markdown/mdast-to-hast-handlers.ts+++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts@@ -1,4 +1,3 @@-import type { Comment, Element, ElementContent, Properties } from 'hast';import type { FootnoteDefinition, FootnoteReference } from 'mdast';import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';diff --git a/packages/core/src/markdown/mdast-to-html.ts b/packages/core/src/markdown/mdast-to-html.tsindex df788a80d..8ce2b2492 100644--- a/packages/core/src/markdown/mdast-to-html.ts+++ b/packages/core/src/markdown/mdast-to-html.ts@@ -1,4 +1,3 @@-import type { Element, Root as HastRoot, Text as HastText } from 'hast';import type { Root as MdastRoot, Text as MdastText } from 'mdast';import type { Handler } from 'mdast-util-to-hast';diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.tsindex fa3cfcd86..bfd7e3109 100644--- a/packages/core/src/markdown/merged-walker.ts+++ b/packages/core/src/markdown/merged-walker.ts@@ -1,4 +1,3 @@-import type { Nodes, Parent, Root } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/mermaid-promoter.ts b/packages/core/src/markdown/mermaid-promoter.tsindex 17ed2b829..1a897fca8 100644--- a/packages/core/src/markdown/mermaid-promoter.ts+++ b/packages/core/src/markdown/mermaid-promoter.ts@@ -1,4 +1,3 @@-import type { Code, Root } from 'mdast';import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';import { visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.tsindex 4aa0490c4..d988f4b8b 100644--- a/packages/core/src/markdown/parse-with-fallback.ts+++ b/packages/core/src/markdown/parse-with-fallback.ts@@ -168,7 +168,6 @@ export function parseRecursive(}}-interface VFilePlace {offset?: number;start?: { offset?: number };@@ -187,7 +186,6 @@ function extractErrorOffset(err: unknown): number | undefined {return undefined;}-interface Region {start: number;end: number;@@ -211,7 +209,6 @@ function nearestBlankLineAfter(src: string, offset: number): number | null {return null;}-export interface TagEvent {kind: 'open' | 'close' | 'self-close';name: string;@@ -358,7 +355,6 @@ function findFallbackRegion(src: string, errorOffset: number): Region {return { start: blockStart, end: blockEnd };}-interface SourceBlock {src: string;start: number;@@ -456,7 +452,6 @@ function tryPerBlockFallback(};}-function wholeDocRawText(source: string): JSONContent {return {type: 'doc',diff --git a/packages/core/src/markdown/parser-drop-closure.ts b/packages/core/src/markdown/parser-drop-closure.tsindex 43d420187..e6ab4fd57 100644--- a/packages/core/src/markdown/parser-drop-closure.ts+++ b/packages/core/src/markdown/parser-drop-closure.ts@@ -1,4 +1,3 @@-export type DroppedTokenAdjudication =| {kind: 'format-dof-axis';diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.tsindex 5af2f05f5..449e6f12f 100644--- a/packages/core/src/markdown/pipeline.ts+++ b/packages/core/src/markdown/pipeline.ts@@ -1,4 +1,3 @@-import {type FromProseMirrorOptions,fromProseMirror,diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.tsindex ccf06e94d..91cdf658f 100644--- a/packages/core/src/markdown/position-aware-join.ts+++ b/packages/core/src/markdown/position-aware-join.ts@@ -1,4 +1,3 @@-import type { Join } from 'mdast-util-to-markdown';import type { Position } from 'unist';import { isBlankLineParagraph } from './zero-emission-blocks.ts';diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.tsindex e5b4c9eba..0f399d34c 100644--- a/packages/core/src/markdown/position-slice.ts+++ b/packages/core/src/markdown/position-slice.ts@@ -1,4 +1,3 @@-import type { Nodes, Root } from 'mdast';import { visit } from 'unist-util-visit';import type { VFile } from 'vfile';@@ -477,8 +476,7 @@ export function applyPositionSliceToNode(const first = source[startOff];if (first !== '[' && first !== '<' && !node.title) {node.data.sourceStyle = 'gfm-autolink';- }- else if (first === '[') {+ } else if (first === '[') {const slice = source.slice(startOff, endOff);const closeBracketIdx = slice.lastIndexOf('](');if (closeBracketIdx !== -1) {diff --git a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.tsindex ca225ce9b..db10fa54b 100644--- a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts+++ b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts@@ -1,4 +1,3 @@-import type { Comment, Element, Root, Text } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.tsindex 3a2524a9a..34068d103 100644--- a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts@@ -1,4 +1,3 @@-import type { Element, ElementContent, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.tsindex 2279a0189..93151d43b 100644--- a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts@@ -1,4 +1,3 @@-import type { Element, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.tsindex a0f467c42..188f02d83 100644--- a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts@@ -1,4 +1,3 @@-import type { Element, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.tsindex 4e8542a21..4ba2c3418 100644--- a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts@@ -1,4 +1,3 @@-import type { Element, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.tsindex a3c96b24a..1a74b1272 100644--- a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts@@ -1,4 +1,3 @@-import type { Element, ElementContent, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.tsindex e95cf43d6..de35e7b79 100644--- a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts@@ -1,4 +1,3 @@-import type { Comment, Element, ElementContent, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.tsindex ffdb4d5b5..b4be1b2ea 100644--- a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts@@ -1,4 +1,3 @@-import type { Element, ElementContent, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.tsindex e2a96d1a7..3c8d88b7c 100644--- a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts+++ b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts@@ -1,4 +1,3 @@-import type { Element, Root } from 'hast';import type { Plugin } from 'unified';diff --git a/packages/core/src/markdown/resolve-image-url.ts b/packages/core/src/markdown/resolve-image-url.tsindex 0389575ac..cf2c9da8e 100644--- a/packages/core/src/markdown/resolve-image-url.ts+++ b/packages/core/src/markdown/resolve-image-url.ts@@ -1,4 +1,3 @@-import { isRelativeUrl } from './safe-url.ts';function isDevDiagnosticContext(): boolean {diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.tsindex 8be632ba4..4d3250d3a 100644--- a/packages/core/src/markdown/safe-url.ts+++ b/packages/core/src/markdown/safe-url.ts@@ -1,4 +1,3 @@-export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');diff --git a/packages/core/src/markdown/serialize-fidelity.test-helper.ts b/packages/core/src/markdown/serialize-fidelity.test-helper.tsindex 96304e7a5..0dd54070e 100644--- a/packages/core/src/markdown/serialize-fidelity.test-helper.ts+++ b/packages/core/src/markdown/serialize-fidelity.test-helper.ts@@ -1,4 +1,3 @@-import type { JSONContent } from '@tiptap/core';import type { Nodes } from 'mdast';import { expect } from 'vitest';diff --git a/packages/core/src/markdown/serialize-helpers.ts b/packages/core/src/markdown/serialize-helpers.tsindex 93798cf83..7cac5b042 100644--- a/packages/core/src/markdown/serialize-helpers.ts+++ b/packages/core/src/markdown/serialize-helpers.ts@@ -1,4 +1,3 @@-import type { Node as PmNode } from '@tiptap/pm/model';import type {MdxJsxAttribute,diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.tsindex 8c08b7a04..b012a5aff 100644--- a/packages/core/src/markdown/single-dollar-math-promoter.ts+++ b/packages/core/src/markdown/single-dollar-math-promoter.ts@@ -1,4 +1,3 @@-import type { PhrasingContent, Root, Text } from 'mdast';import type { InlineMath } from 'mdast-util-math';import { SKIP, visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.tsindex 959cc0bd2..3e79f8cb6 100644--- a/packages/core/src/markdown/to-markdown-handlers.ts+++ b/packages/core/src/markdown/to-markdown-handlers.ts@@ -1,4 +1,3 @@-import type { Nodes, Parents } from 'mdast';import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';import type { Handle, Info, State } from 'mdast-util-to-markdown';diff --git a/packages/core/src/markdown/underline-promoter.ts b/packages/core/src/markdown/underline-promoter.tsindex 6e6d73400..0f1b0f27d 100644--- a/packages/core/src/markdown/underline-promoter.ts+++ b/packages/core/src/markdown/underline-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Parent, Root } from 'mdast';import type { MdxJsxTextElement } from 'mdast-util-mdx';import { visit } from 'unist-util-visit';diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.tsindex 932d7d2a8..1a082bda7 100644--- a/packages/core/src/markdown/unknown-mdast-guard.ts+++ b/packages/core/src/markdown/unknown-mdast-guard.ts@@ -1,4 +1,3 @@-import type { Root as MdastRoot } from 'mdast';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.tsindex ac9f5239b..d578476a7 100644--- a/packages/core/src/markdown/whitespace-char-ref.ts+++ b/packages/core/src/markdown/whitespace-char-ref.ts@@ -1,4 +1,3 @@-const INLINE_WHITESPACE_BY_CODE: ReadonlyMap<number, string> = new Map([[0x20, ' '],[0x09, '\t'],diff --git a/packages/core/src/markdown/wiki-link-micromark.ts b/packages/core/src/markdown/wiki-link-micromark.tsindex afd9936fe..6dad26200 100644--- a/packages/core/src/markdown/wiki-link-micromark.ts+++ b/packages/core/src/markdown/wiki-link-micromark.ts@@ -17,7 +17,6 @@ declare module 'micromark-util-types' {}}-const CODE_BANG = 33; // !const CODE_LBRACKET = 91; // [const CODE_RBRACKET = 93; // ]@@ -261,7 +260,6 @@ export function wikiLinkSyntax(): Extension {};}-function enterWikiLink(this: CompileContext, token: Token) {this.enter({@@ -387,7 +385,6 @@ export const wikiLinkToMarkdown: {unsafe: [{ character: '[', inConstruct: ['phrasing'] }],};-const MICROMARK_EXT = wikiLinkSyntax();export function remarkWikiLink(this: Processor) {diff --git a/packages/server/src/api-asset-text.test.ts b/packages/server/src/api-asset-text.test.tsnew file mode 100644index 000000000..c69ec66c6--- /dev/null+++ b/packages/server/src/api-asset-text.test.ts@@ -0,0 +1,154 @@+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import type { IncomingMessage, ServerResponse } from 'node:http';+import { createServer, type Server } from 'node:http';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { createApiExtension } from './api-extension.test-helper.ts';+import type { ContentFilter } from './content-filter.ts';+import { listenOnLoopback } from './loopback-rig-test-helpers.ts';++/**+ * Characterization for GET /api/asset-text ahead of the assets-service+ * extraction — this endpoint had no direct coverage. The pins mirror the+ * documented posture that deliberately DIFFERS from /api/asset: any+ * extension is admitted (no ASSET_EXTENSIONS gate) and ignored files are+ * served (no contentFilter gate) — path containment is the load-bearing+ * check, plus the 1 MiB viewer cap.+ */++interface Harness {+ baseURL: string;+ close: () => Promise<void>;+}++async function startHarness(contentDir: string, contentFilter?: ContentFilter): Promise<Harness> {+ const ext = createApiExtension({+ hocuspocus: {} as Parameters<typeof createApiExtension>[0]['hocuspocus'],+ sessionManager: {} as Parameters<typeof createApiExtension>[0]['sessionManager'],+ contentDir,+ serverInstanceId: 'test-server',+ getFileIndex: () => new Map(),+ contentFilter,+ });++ const server: Server = createServer((req, res) => {+ void (+ ext as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ }+ ).onRequest({ request: req, response: res });+ });++ const { baseUrl } = await listenOnLoopback(server);++ return {+ baseURL: baseUrl,+ close: () =>+ new Promise<void>((resolve, reject) => {+ server.close((err) => (err ? reject(err) : resolve()));+ }),+ };+}++function assetTextUrl(baseURL: string, path: string): string {+ return `${baseURL}/api/asset-text?path=${encodeURIComponent(path)}`;+}++describe('GET /api/asset-text', () => {+ let tmpDir: string;+ let contentDir: string;+ let harness: Harness;++ beforeEach(async () => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-api-asset-text-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(join(contentDir, 'docs'), { recursive: true });+ writeFileSync(join(contentDir, 'docs', 'data.csv'), 'a,b\n1,2\n');+ writeFileSync(join(contentDir, 'docs', '.DS_Store'), 'dotfile-bytes');+ writeFileSync(join(contentDir, 'docs', 'photo.png'), 'fake-png-bytes');+ writeFileSync(join(contentDir, 'docs', 'huge.log'), 'x'.repeat(1_048_577));+ writeFileSync(join(contentDir, 'docs', 'ignored.txt'), 'hidden but readable');+ mkdirSync(join(contentDir, 'docs', 'somedir'));+ writeFileSync(join(tmpDir, 'outside.txt'), 'outside');+ symlinkSync(join(tmpDir, 'outside.txt'), join(contentDir, 'docs', 'escape.txt'));+ const filter = {+ isPathIgnored: (p: string) => p === 'docs/ignored.txt',+ } as unknown as ContentFilter;+ harness = await startHarness(contentDir, filter);+ });++ afterEach(async () => {+ await harness.close();+ rmSync(tmpDir, { recursive: true, force: true });+ });++ test('serves any extension as forced text/plain with the viewer headers', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/data.csv'));+ expect(res.status).toBe(200);+ expect(res.headers.get('content-type')).toBe('text/plain; charset=utf-8');+ expect(res.headers.get('content-disposition')).toBe('inline');+ expect(res.headers.get('x-content-type-options')).toBe('nosniff');+ expect(res.headers.get('cache-control')).toBe('no-store');+ expect(await res.text()).toBe('a,b\n1,2\n');+ });++ test('no extension gate: dotfiles and binary-extension files are served as text', async () => {+ const dotfile = await fetch(assetTextUrl(harness.baseURL, 'docs/.DS_Store'));+ expect(dotfile.status).toBe(200);+ expect(await dotfile.text()).toBe('dotfile-bytes');++ const png = await fetch(assetTextUrl(harness.baseURL, 'docs/photo.png'));+ expect(png.status).toBe(200);+ expect(png.headers.get('content-type')).toBe('text/plain; charset=utf-8');+ });++ test('no ignore-filter gate: an ignored file is still served (unlike /api/asset)', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/ignored.txt'));+ expect(res.status).toBe(200);+ expect(await res.text()).toBe('hidden but readable');+ });++ test('missing path is a 400 invalid-request', async () => {+ const res = await fetch(`${harness.baseURL}/api/asset-text`);+ expect(res.status).toBe(400);+ const body = (await res.json()) as { type: string };+ expect(body.type).toBe('urn:ok:error:invalid-request');+ });++ test('nonexistent file is a 404 asset-not-found', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/nope.txt'));+ expect(res.status).toBe(404);+ const body = (await res.json()) as { type: string };+ expect(body.type).toBe('urn:ok:error:asset-not-found');+ });++ test('a directory is a 404, not a listing', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/somedir'));+ expect(res.status).toBe(404);+ });++ test('symlink escaping contentDir is a 400 invalid-request', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/escape.txt'));+ expect(res.status).toBe(400);+ const body = (await res.json()) as { type: string };+ expect(body.type).toBe('urn:ok:error:invalid-request');+ });++ test('non-canonical traversal spelling is a 400 even when it stays inside', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/../docs/data.csv'));+ expect(res.status).toBe(400);+ });++ test('files over the 1 MiB viewer cap are a 413', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/huge.log'));+ expect(res.status).toBe(413);+ const body = (await res.json()) as { type: string };+ expect(body.type).toBe('urn:ok:error:payload-too-large');+ });++ test('non-GET is a 405', async () => {+ const res = await fetch(assetTextUrl(harness.baseURL, 'docs/data.csv'), { method: 'POST' });+ expect(res.status).toBe(405);+ });+});diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.tsindex 8a1d1a34e..844015fa6 100644--- a/packages/server/src/api-extension.ts+++ b/packages/server/src/api-extension.ts@@ -7,9 +7,8 @@*/import { type SpawnOptions, spawn } from 'node:child_process';-import { createHash, randomUUID } from 'node:crypto';+import { randomUUID } from 'node:crypto';import {- closeSync,createReadStream,createWriteStream,type Dirent,@@ -17,10 +16,8 @@ import {lstatSync,mkdirSync,mkdtempSync,- openSync,readdirSync,readFileSync,- readSync,realpathSync,statSync,unlinkSync,@@ -49,7 +46,6 @@ import {AgentWriteRequestSchema,AgentWriteSuccessSchema,ApiConfigSuccessSchema,- ASSET_EXTENSIONS,applyPatchToFm,BacklinkCountsSuccessSchema,BacklinksSuccessSchema,@@ -72,10 +68,6 @@ import {colorFromSeed,composeWithDerivedFrontmatter,createCodeFenceTracker,- createWorkspaceSearchCorpus,- createWorkspaceSearchDocument,- DEFAULT_ATTACHMENT_FOLDER_PATH,- DEFAULT_DEDUP_MODE,DEFAULT_EMBEDDINGS_BASE_URL,DEFAULT_EMBEDDINGS_MODEL,DEFAULT_LINKS_VALIDATION,@@ -91,7 +83,6 @@ import {DuplicatePathSuccessSchema,detectFmRegion,EDITOR_PROJECT_SKILL_ROOT,- EDITOR_USER_SKILL_ROOT,type EditorId,EmbedDetectSuccessSchema,EmptyRequestSchema,@@ -110,7 +101,6 @@ import {HistorySuccessSchema,HistoryVersionSuccessSchema,HubsSuccessSchema,- INLINE_RENDERABLE_EXTENSIONS,type InlineAssetMediaKind,InstallSkillRequestSchema,InstallSkillSuccessSchema,@@ -121,7 +111,6 @@ import {isManagedArtifactDocName,isOpenKnowledgeSkillsSource,isSkillInstallTarget,- isValidAttachmentFolderPath,LEGACY_SKILL_STORE_ROOT,LINKABLE_ASSET_EXTENSIONS,type LifecycleStatus,@@ -165,7 +154,6 @@ import {type MetricsWatcherRecentSuccess,MetricsWatcherRecentSuccessSchema,mediaKindForSidebarAssetExtension,- normalizeAttachmentFolderPath,OK_DIR,OPENKNOWLEDGE_SKILLS_REPO,OrphansSuccessSchema,@@ -176,7 +164,6 @@ import {PrincipalSuccessSchema,type ProblemType,parseFrontmatterRecord,- parseProjectSkillBundleDoc,parseTemplateFile,prependFrontmatter,projectSkillContentDocName,@@ -189,14 +176,10 @@ import {RollbackRequestSchema,RollbackSuccessSchema,readFmMap,- SANDBOXED_HTML_CSP,- SANDBOXED_HTML_EXTENSIONS,SaveVersionRequestSchema,SaveVersionSuccessSchema,SearchRequestSchema,- type SearchSemanticStatus,type SearchSource,- type SearchSuccess,SearchSuccessSchema,SeedApplyRequestSchema,SeedApplySuccessSchema,@@ -225,7 +208,6 @@ import {SkillFileRenameRequestSchema,SkillFileRenameSuccessSchema,SkillGetSuccessSchema,- type SkillHostIdArg,type SkillImportBulkResult,SkillImportRequestSchema,SkillImportSuccessSchema,@@ -265,7 +247,6 @@ import {SyncTriggerRequestSchema,SyncTriggerSuccessSchema,scanHeadingLine,- searchWorkspaceCorpus,skillLiveDocName,stripFrontmatter,TagsForNameSuccessSchema,@@ -288,17 +269,12 @@ import {UploadAssetSuccessSchema,UploadRequestSchema,unwrapFrontmatterFences,- updateWorkspaceSearchCorpus,ValidationAuditCountsResponseSchema,ValidationAuditResponseSchema,type ValidationDiagnostic,- type WorkspaceSearchCorpus,- type WorkspaceSearchDocument,type WorkspaceSearchIntent,type WorkspaceSearchRanking,- type WorkspaceSearchResult,type WorkspaceSearchScope,- type WorkspaceSemanticInput,WorkspaceSuccessSchema,} from '@inkeep/open-knowledge-core';import {@@ -312,8 +288,6 @@ import {discoverSkillDirs,enumerateInstalledSkills,fetchSource,- findByContentHash,- inspectPluginSource,parseSkillDir,parseSkillsLock,parseSource,@@ -323,9 +297,6 @@ import {resolvePluginUpdateSource,resolveSkillsShImportSource,retrofitPackLockEntry,- SKILL_IMPORT_MAX_BUNDLE_FILES,- SKILL_IMPORT_MAX_FILE_BYTES,- SKILL_IMPORT_MAX_TOTAL_BYTES,SKILLS_LOCK_REL,SkillFetchError,type SkillsLock,@@ -334,7 +305,6 @@ import {type WellKnownIndex,} from '@inkeep/open-knowledge-core/skills-catalog';import busboy from 'busboy';-import { fileTypeFromBuffer } from 'file-type';import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';import { type Entry, fromBuffer as yauzlFromBuffer, type ZipFile } from 'yauzl';import { z } from 'zod';@@ -369,8 +339,7 @@ import {} from './agent-sessions.ts';import { type NormalizedSummary, normalizeSummary } from './agent-write-summary.ts';import { isAllowedApiOrigin } from './api-origin.ts';-import { collectReferencedAssets, toContentRelativePath } from './asset-references.ts';-import { assetContentTypeForPath } from './asset-serve-middleware.ts';+import { collectReferencedAssets } from './asset-references.ts';import { collabUrlFromRequestHeaders } from './collab-bootstrap-url.ts';import { createCommentApi } from './comments/comment-api.ts';import { CommentIndex } from './comments/comment-index.ts';@@ -411,16 +380,11 @@ import {} from './content-divergence-gate.ts';import { recordContributor } from './contributor-tracker.ts';import { deriveDetection, embedProbeRing, recordEmbedProbe } from './embed-probe.ts';-import {- recordSemanticQuery,- type SemanticQueryOutcome,-} from './embeddings/embeddings-telemetry.ts';import {FileEmbeddingsBackend,probeEmbeddingEndpoint,type ResolvedSemanticConfig,resolveEmbeddingsCredential,- SEMANTIC_MIN_QUERY_LENGTH,type SemanticSearchService,} from './embeddings/index.ts';import {@@ -428,6 +392,7 @@ import {FrontmatterMalformedError,respondFrontmatterMalformed,} from './frontmatter-malformed-error.ts';+import { assertNoSymlinkEscape } from './fs-safety.ts';import {createInstalledAgentsProbe,createOsProbe,@@ -528,16 +493,12 @@ import {} from './skill-bundles.ts';import { buildAndOpenSkill, detectUserSkillHosts } from './skill-install.ts';import {- classifyInPlaceDest,- projectInPlaceSkill,projectSkill,readSkillBundledFiles,- relocateInPlaceCanonical,removeInPlaceSkillCopies,resolvedHosts,resolveSkillTargets,reverseProjectSkill,- skillHostDir,skillProjectionRoots,validateSkillForInstall,} from './skill-projection.ts';@@ -546,11 +507,7 @@ import { readSkillInstallStateSnapshot } from './skill-state.ts';import { handleSpawnCursor } from './spawn-cursor-api.ts';import { assertRealpathWithinDir } from './symlink-guard.ts';import { readUiLock } from './ui-lock.ts';-import {- HashingPassThrough,- linkTempToFinalWithCollisionRetry,- mintTempUploadPath,-} from './upload-streaming.ts';+import { HashingPassThrough, mintTempUploadPath } from './upload-streaming.ts';/** Does the bundle at `dir` carry a `metadata.pack` marker in its frontmatter? */function bundleSelfIdentifiesAsPack(dir: string): boolean {@@ -600,8 +557,12 @@ import type { BridgeDeriveLossReporter } from './bridge-loss-detector.ts';import { isConfigDoc, isLinkIndexExcludedDoc, isSystemDoc } from './cc1-broadcast.ts';import { withHiddenWindowsConsole } from './child-process-windows-hide.ts';import type { ResolveStrategy } from './conflict-storage.ts';+import {+ isReservedProjectStatePath,+ listManagedDocNamesUnderFolder,+} from './content/managed-doc-enum.ts';import type { ContentFilter } from './content-filter.ts';-import { isWithinContentDir, safeContentPath } from './content-path.ts';+import { safeContentPath } from './content-path.ts';import {type DerivedDocumentIndexApiPort,type DerivedDocumentIndexMutation,@@ -646,9 +607,7 @@ import {tracedCpSync,tracedMkdirSync,tracedRenameSync,- tracedRmdirSync,tracedRmSync,- tracedSymlinkSync,tracedUnlinkSync,tracedWriteFileSync,} from './fs-traced.ts';@@ -736,7 +695,6 @@ import { hostHeaderMatchesPublicHost } from './remote-access.ts';import {appendRenameLogEntry,createAncestorShaSetCache,- gcRenameLog,getOrLoadRenameLogIndex,type RenameLogEntry,resolveDocPathAtCommit,@@ -751,13 +709,23 @@ import {SeedRootDirError,} from './seed/index.ts';import type { PairedWriteOrigin } from './server-observers.ts';+import { createAssetService } from './services/assets.ts';+import { createFileOpsService } from './services/file-ops.ts';+import { createSearchService } from './services/search.ts';+import {+ createSkillImportService,+ importedBundleLimitError,+ SKILL_IMPORT_WRITE_LIMITS,+ type SkillImportOutcome,+} from './services/skill-import.ts';+import { createSkillInstallOpsService } from './services/skill-install-ops.ts';+import { createSkillPlacementOpsService } from './services/skill-placement-ops.ts';+import { createVersionOpsService } from './services/version-ops.ts';import {- enumerateWipChains,listRescueCheckpoints,SERVICE_WRITER,type ShadowRef,safetyCheckpoint,- saveVersion,shadowGit,type TimelineRescueEntry,type WriterIdentity,@@ -776,10 +744,6 @@ import {readSkillPlacements,recordFolderExpectation,recordKnownSkillRoot,- recordSkillPlacement,- recordSkillSourceHost,- removeSkillPlacement,- resolveSkillPlacementPath,} from './skill-placements.ts';import { restoreSkillVersion } from './skill-restore.ts';import { mutateSkillsLock, readSkillsLockFile } from './skills-lock-store.ts';@@ -797,30 +761,6 @@ import { recordTimelineCoalesced } from './timeline-telemetry.ts';// instrument on every hit.let _httpDurationHist: ReturnType<ReturnType<typeof getMeter>['createHistogram']> | null = null;-const SKILL_IMPORT_WRITE_LIMITS = {- maxFileBytes: SKILL_IMPORT_MAX_FILE_BYTES,- maxFiles: SKILL_IMPORT_MAX_BUNDLE_FILES,-} as const;--function importedBundleLimitError(- acquired: NonNullable<ReturnType<typeof parseSkillDir>>,-): string | null {- if (acquired.files.length > SKILL_IMPORT_MAX_BUNDLE_FILES) {- return `Skill has ${acquired.files.length} dependent files; the import cap is ${SKILL_IMPORT_MAX_BUNDLE_FILES}.`;- }- let totalBytes = Buffer.byteLength(acquired.skillMd);- for (const file of acquired.files) {- const bytes = file.bytes?.byteLength ?? Buffer.byteLength(file.content ?? '');- if (bytes > SKILL_IMPORT_MAX_FILE_BYTES) {- return `${file.relPath} is ${bytes} bytes; the import per-file cap is ${SKILL_IMPORT_MAX_FILE_BYTES}.`;- }- totalBytes += bytes;- }- return totalBytes > SKILL_IMPORT_MAX_TOTAL_BYTES- ? `Skill is ${totalBytes} bytes; the import bundle cap is ${SKILL_IMPORT_MAX_TOTAL_BYTES}.`- : null;-}-function httpDurationHist(): ReturnType<ReturnType<typeof getMeter>['createHistogram']> {_httpDurationHist ||= getMeter().createHistogram('http.server.request.duration', {description: 'HTTP server request duration in seconds',@@ -927,36 +867,6 @@ function agentWriteContentDivergenceCounter(): ReturnType<return _agentWriteContentDivergenceCounter;}-// Counter for the name-only `kind:'file'` corpus tier hitting the-// `OK_SEARCH_MAX_ENTRIES` cap. Increments once per corpus rebuild that drops-// deepest-tail paths; the matching warn log carries the (dropped, retained,-// limit) breakdown. Lets an operator distinguish "cap fired" from "results-// quietly missing" without scraping logs.-let _searchCorpusTruncatedCounter: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null =- null;-function searchCorpusTruncatedCounter(): ReturnType<ReturnType<typeof getMeter>['createCounter']> {- _searchCorpusTruncatedCounter ||= getMeter().createCounter('ok.search.corpus_truncated_total', {- description:- 'Count of search-corpus rebuilds where the name-only file tier hit OK_SEARCH_MAX_ENTRIES and dropped deepest-tail paths. One increment per truncated build; non-truncated builds do not increment.',- });- return _searchCorpusTruncatedCounter;-}--// Counter for search-corpus builds by mode: `cold` (no prior corpus),-// `incremental` (in-place index patch of only the changed documents), or-// `rebuild` (incremental path fell back to a from-scratch build; the bounded-// `reason` attribute says why). A steady stream of `rebuild` where-// `incremental` is expected is the drift signal worth alerting on.-let _searchCorpusUpdateCounter: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null =- null;-function searchCorpusUpdateCounter(): ReturnType<ReturnType<typeof getMeter>['createCounter']> {- _searchCorpusUpdateCounter ||= getMeter().createCounter('ok.search.corpus_update_total', {- description:- 'Count of workspace search corpus builds, by mode (cold | incremental | rebuild) and, for rebuilds, the bounded fallback reason.',- });- return _searchCorpusUpdateCounter;-}-/** Bounded handler label for the content-divergence counters. */type DivergenceHandler = 'agent-write-md' | 'agent-write-batch' | 'agent-patch' | 'rollback';@@ -1126,239 +1036,8 @@ function docTreePathCandidates(docName: string, contentRoot: string): readonly sreturn extless ? [p.path, extless] : [p.path];}-const GENERIC_PASTE_NAMES = /^(image\.(png|jpe?g|gif|webp)|Clipboard.*|Untitled.*)$/i;--// unicode-preserving. Permits any Unicode letter, number, or combining-// mark, plus pictographic emoji and the punctuation whitelist (., -, _, space).-// Everything else (including `/`, `\`, null bytes, control chars, CRLF) is-// either stripped or replaced so path-escape guards downstream keep their-// invariants. CJK, Arabic, Cyrillic, and emoji survive — macOS/Finder-// ergonomics without sacrificing filesystem safety.-const SAFE_FILENAME_CHARS = /[^\p{L}\p{N}\p{M}\p{Extended_Pictographic}.\-_ ]/gu;-// Stripping C0 + DEL is the whole point — the rule fires on intentional use.-// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional — sanitize must strip control bytes.-const STRIP_ON_SIGHT = /[/\\\x00-\x1f\x7f]/g;--export function sanitizeFilename(name: string): string {- // Strip path separators and null/control bytes BEFORE any other pass so- // they cannot reappear inside a replacement and dodge later checks.- let stripped = name.replace(STRIP_ON_SIGHT, '');- stripped = stripped.replace(SAFE_FILENAME_CHARS, '_');-- // Collapse underscore and dot runs so "../etc/passwd" → "etcpasswd" and- // "foo__bar" → "foo_bar".- stripped = stripped.replace(/_+/g, '_').replace(/\.{2,}/g, '.');-- // No hidden files — trim leading dots and leading underscores.- stripped = stripped.replace(/^[._]+/, '');- // Filesystem portability — strip trailing dots (Windows trims them too).- stripped = stripped.replace(/\.+$/, '');-- if (stripped === '') return 'upload';-- // Most filesystems cap basenames at 255 bytes (ext4, APFS, exFAT). Without a- // ceiling, a multipart `Content-Disposition` filename approaching busboy's- // header size can sail through Unicode-letter sanitization and surface as- // `ENAMETOOLONG` from `linkSync`, which classifies as a generic- // `storage-error` → 500. Truncate the stem (preserving the extension) to- // stay within the portable basename ceiling.- const MAX_BYTES = 255;- const encoder = new TextEncoder();- if (encoder.encode(stripped).length > MAX_BYTES) {- const dotIdx = stripped.lastIndexOf('.');- const ext = dotIdx >= 0 ? stripped.slice(dotIdx) : '';- let stem = dotIdx >= 0 ? stripped.slice(0, dotIdx) : stripped;- // `slice(0, -1)` removes one UTF-16 code unit. A trailing emoji is a- // surrogate pair, so the loop transiently produces a lone-surrogate- // string that `TextEncoder` re-encodes as U+FFFD (3 bytes) — harmless- // since the emoji is fully consumed before the loop exits and the- // returned string is always valid UTF-8.- while (encoder.encode(stem + ext).length > MAX_BYTES && stem.length > 0) {- stem = stem.slice(0, -1);- }- stripped = (stem || 'upload') + ext;- // The loop drains the stem; it cannot shrink the extension itself.- // An adversarial 250+ byte extension (e.g. `'x.' + 'a'.repeat(300)`)- // would drain the stem to empty and still leave `'upload' + ext`- // above the ceiling. Final-pass guard: fall back to extensionless- // `'upload'` when even the floor exceeds MAX_BYTES.- if (encoder.encode(stripped).length > MAX_BYTES) stripped = 'upload';- }-- return stripped;-}--/**- * Resolve the destination directory for an upload from the parent doc's- * path and the configured `content.attachmentFolderPath`. Matches Obsidian's- * literal schema (free-form string):- *- * - `"./"` (default) → same directory as the doc- * - `"/"` → content-directory root- * - `"./<sub>"` → subdirectory beside the doc- * - `"<name>"` (bare) → fixed content-relative path- *- * Treats any `./` prefix as "relative to doc dir," any other value as- * "relative to content dir." Empty or whitespace-only strings fall back- * to the default (doc dir).- *- * Returns an absolute path within `resolvedContentDir` — path-escape- * enforcement happens at the caller via `isWithinContentDir` + `realpath`.- */-export function resolveUploadDestDir(- parentDocName: string,- attachmentFolderPath: string,- resolvedContentDir: string,-): string {- const trimmed = attachmentFolderPath.trim();- if (trimmed === '' || trimmed === './') {- return resolve(resolvedContentDir, dirname(parentDocName));- }- if (trimmed === '/') {- return resolvedContentDir;- }- if (trimmed.startsWith('./')) {- // Subdirectory beside the doc. `"./attachments"` → `<docDir>/attachments`.- return resolve(resolvedContentDir, dirname(parentDocName), trimmed.slice(2));- }- // Bare name or nested path: fixed content-relative location.- return resolve(resolvedContentDir, trimmed);-}--/**- * Read at most `n` bytes from the start of `path`. Feeds both the magic-byte- * sniff (`fileTypeFromBuffer` over the head) and the SVG text fallback- * (`file-type` can't detect text-based SVG), without ever materializing the- * whole file.- */-function readTempFileHead(path: string, n: number): Buffer {- const fd = openSync(path, 'r');- try {- const buf = Buffer.alloc(n);- const read = readSync(fd, buf, 0, n, 0);- return buf.subarray(0, read);- } finally {- closeSync(fd);- }-}--/**- * Scan `destDir` non-recursively for an existing file whose sha256 matches- * the buffer's. Returns the matching basename (case-preserving) or null if- * no match. Bounded by directory size — O(n) in sibling count, not vault size.- * Only files with extensions in ASSET_EXTENSIONS are candidates; everything- * else (markdown, .git/, etc.) is skipped.- *- * `expectedSize` is the buffer's byte length — passed in so we can size-- * prefilter before hashing siblings. sha256 collision requires equal-sized- * inputs, so same-extension siblings with a different size are not- * candidates and we skip their (potentially multi-MB) read. This turns- * the common "paste a new screenshot" path from O(total asset bytes in- * dir) back to O(sibling count × stat). Non-ENOENT read failures log at- * WARN so silent dedup degradation has a signal.- */-/**- * Upper bound on size-matched candidates we'll read+hash in a single- * dedup call. A capture-device folder with 1000+ screenshots at the same- * resolution could theoretically produce that many same-size siblings;- * each candidate costs a sync readFileSync + sha256Hex of the entire- * buffer, which would block the event loop for seconds per upload under- * adversarial / pathological load.- *- * Past the bound, dedup degrades to best-effort: we log a structured- * WARN and return null (treat as no-match → write a new file with the- * collision-suffix loop). This is a bounded-resource defense, not a- * correctness change — a duplicate that slips through produces the- * cheap storage cost of one extra on-disk copy, not silent data loss.- * The O(1) hash-cache alternative is a- * larger architectural change and a follow-on.- */-const MAX_DEDUP_SCAN_CANDIDATES = 1000;--/**- * Stream a file's bytes through a sha256 Hash transform and return the hex- * digest. Keeps memory O(1) regardless of file size — a 500 MB candidate- * read by the buffer-based `readFileSync` path would otherwise materialize- * the whole file in heap, which defeats the streaming-upload amendment's- * O(1) memory guarantee.- *- * Throws on read errors so the caller can classify ENOENT (concurrent- * rename — stay silent) vs other errors (log and skip).- */-async function streamingHashFile(path: string): Promise<string> {- const hash = createHash('sha256');- await pipeline(createReadStream(path), hash);- return hash.digest('hex');-}--async function findDuplicateAsset(- destDir: string,- sha: string,- expectedSize: number,-): Promise<string | null> {- let entries: string[];- try {- // Async `readdir` so the directory walk doesn't block the event- // loop during uploads — Node's event loop is shared with WebSocket sync- // and CRDT updates, and a 1k-entry walk is observable on bursty- // upload traffic. The MAX_DEDUP_SCAN_CANDIDATES cap- // bounds the worst case at 1000 same-size siblings, but the- // pre-cap entry list can still be much larger.- entries = await readdir(destDir);- } catch {- return null;- }- const log = getLogger('upload');- let scanned = 0;- for (const entry of entries) {- const ext = extname(entry).slice(1).toLowerCase();- if (!ASSET_EXTENSIONS.has(ext)) continue;- const fullPath = resolve(destDir, entry);- let entryStat: Awaited<ReturnType<typeof stat>>;- try {- entryStat = await stat(fullPath);- } catch {- continue;- }- if (!entryStat.isFile() || entryStat.size !== expectedSize) continue;- // Bounded scan: only count candidates that passed the cheap size- // prefilter, since same-size siblings are the ones that cost a- // full-file hash each (streaming now, not buffered).- scanned++;- if (scanned > MAX_DEDUP_SCAN_CANDIDATES) {- log.warn(- {- event: 'upload-dedup-skip',- reason: 'scan-cap-exceeded',- destDir,- scanned: MAX_DEDUP_SCAN_CANDIDATES,- expectedSize,- },- `[upload-dedup] candidate scan exceeded ${MAX_DEDUP_SCAN_CANDIDATES} same-size siblings — degrading to no-dedup for this upload`,- );- return null;- }- let candidateSha: string;- try {- // Stream + hash the candidate to preserve the O(1) memory guarantee- // the upload pipeline otherwise maintains end-to-end. A 500 MB- // candidate otherwise spiked heap to 500 MB per scan.- candidateSha = await streamingHashFile(fullPath);- } catch (err) {- const code = errnoCode(err);- // ENOENT is the legitimate concurrent-rename race — stay silent.- if (code !== 'ENOENT') {- log.warn(- { event: 'upload-dedup-skip', reason: 'read-failed', code, entry },- '[upload-dedup] skipped candidate — read failed',- );- }- continue;- }- if (candidateSha === sha) return entry;- }- return null;-}+export { sanitizeFilename } from './filename-sanitize.ts';+export { resolveUploadDestDir } from './services/assets.ts';/*** Discriminator for write failures so the upload handler can surface a@@ -2266,12 +1945,6 @@ function isValidRelativeContentPath(path: string): boolean {* externally-addressed `.OK/x` IS `.ok/x`. Same segment walk as* `pathHasAlwaysSkipSegment` in content-filter.ts.*/-function isReservedProjectStatePath(path: string): boolean {- return path.split('/').some((segment) => {- const normalized = segment.toLowerCase();- return normalized === OK_DIR || normalized === '.git';- });-}function isReservedSyntheticFolderPath(path: string): boolean {return (@@ -2333,58 +2006,6 @@ function requireNonEmptyDocName(return null;}-/**- * Ensures `fullPath` does not escape `resolvedContentDir` via symlinks (matches persistence- * symlink-escape checks). Walks up with dirname when the leaf is missing so destinations like- * `link/new.md` are rejected if `link` resolves outside the content dir.- *- * Uses `realpathSync(resolvedContentDir)` as the boundary anchor so platform normalization- * (e.g. macOS `/var` → `/private/var`) matches `realpathSync` of paths under it.- */-function assertNoSymlinkEscape(fullPath: string, resolvedContentDir: string): void {- let contentRoot: string;- try {- contentRoot = realpathSync(resolvedContentDir);- } catch (err) {- const code = errnoCode(err);- // ENOENT means the content dir hasn't been created yet — no symlink- // escape is possible against a non-existent directory, but we have- // no safe baseline for the check either. Throw the same- // `symlink-escape:` error class so the caller's catch routes through- // the existing error path. Other errno classes (EPERM, EIO, ENOMEM)- // must NOT be swallowed silently — they'd leave the security gate- // disabled with no log line, no telemetry, no error response. Throw- // and let the top-level handler emit a typed RFC 9457 problem.- if (code === 'ENOENT') {- throw new SymlinkEscapeError('content directory does not exist');- }- throw err;- }-- let cur = fullPath;- for (;;) {- try {- const canonical = realpathSync(cur);- if (!isWithinContentDir(canonical, contentRoot)) {- throw new SymlinkEscapeError('path resolves outside content directory');- }- return;- } catch (err) {- const code = errnoCode(err);- if (code === 'ELOOP') {- throw new SymlinkEscapeError('symlink cycle in path');- }- if (code !== 'ENOENT') throw err;- const parent = dirname(cur);- if (parent === cur) throw err;- if (parent !== resolvedContentDir && !parent.startsWith(`${resolvedContentDir}${sep}`)) {- throw err;- }- cur = parent;- }- }-}-function resolveContentEntryPath(contentDir: string, kind: ContentEntryKind, path: string): string {if (!isValidRelativeContentPath(path)) {throw new Error('path must be a relative content path');@@ -2432,21 +2053,6 @@ class DuplicateNameExhaustedError extends Error {}}-function isAlreadyExistsError(err: unknown): boolean {- const code = errnoCode(err);- // Node's cpSync distinguishes the destination-occupied cases by the type- // mismatch: a same-type collision surfaces as ERR_FS_CP_EEXIST, but copying a- // directory onto an existing file (or a file onto an existing directory)- // surfaces as ERR_FS_CP_DIR_TO_NON_DIR / ERR_FS_CP_NON_DIR_TO_DIR. All three- // mean the destination path is already taken and must map to 409.- return (- code === 'EEXIST' ||- code === 'ERR_FS_CP_EEXIST' ||- code === 'ERR_FS_CP_DIR_TO_NON_DIR' ||- code === 'ERR_FS_CP_NON_DIR_TO_DIR'- );-}-type DuplicatePathFilesystemProblem = {status: 500 | 507;type: Extract<ProblemType, 'urn:ok:error:storage-full' | 'urn:ok:error:storage-readonly'>;@@ -3165,16 +2771,6 @@ export interface ApiExtensionOptions {getLinksValidationSetting?: () => LinksValidationSetting;}-interface WorkspaceSearchCacheEntry {- fingerprint: string;- corpus?: WorkspaceSearchCorpus;- /** Whether the name-only file tier hit `OK_SEARCH_MAX_ENTRIES` on this build. */- truncated?: boolean;- pending?: Promise<{ corpus: WorkspaceSearchCorpus; truncated: boolean }>;-}--const workspaceSearchCaches = new Map<string, WorkspaceSearchCacheEntry>();-/*** Extract all ATX headings (# … ######) from a Markdown document.* Frontmatter is stripped before scanning so `title:` YAML lines are ignored.@@ -4459,46 +4055,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {* `ManagedRenameMissingDocumentError`, or a link rewrite would write a `.md`* sibling of the moved `.mdx` (split-brain).*/- function listManagedDocNamesUnderFolderFromDisk(sourcePathRoot: string): string[] {- const docNames: string[] = [];- // A file at the folder path (e.g. `kind: 'folder'` on a doc) must NOT reach- // `readdirSync` — that throws ENOTDIR and 500s. Return empty for that and a- // TOCTOU vanish (ENOENT) so the caller's type-mismatch / not-found check- // emits the correct 4xx. Any other stat error (EACCES, EIO, ELOOP) means- // the folder exists but is unreadable: returning empty there would move the- // directory and skip link rewriting — the exact bug this fix addresses — so- // rethrow and let it surface as a 500.- try {- if (!statSync(sourcePathRoot).isDirectory()) return docNames;- } catch (err) {- const code = errnoCode(err);- if (code === 'ENOENT' || code === 'ENOTDIR') return docNames;- throw err;- }-- function walk(dir: string): void {- for (const entry of readdirSync(dir, { withFileTypes: true })) {- const fullPath = resolve(dir, entry.name);- const relPath = relative(contentDir, fullPath).split(sep).join('/');- if (isReservedProjectStatePath(relPath)) continue;- if (entry.isDirectory()) {- if (contentFilter?.isDirExcluded(relPath)) continue;- walk(fullPath);- continue;- }- if (!entry.isFile() || !isSupportedDocFile(relPath) || contentFilter?.isExcluded(relPath)) {- continue;- }- const docName = docNameForFileOperationPath(contentDir, relPath);- registerDocExtension(stripDocExtension(relPath), extname(relPath));- docNames.push(docName);- }- }-- walk(sourcePathRoot);- docNames.sort((a, b) => a.localeCompare(b));- return docNames;- }+ const listManagedDocNamesUnderFolderFromDisk = (sourcePathRoot: string): string[] =>+ listManagedDocNamesUnderFolder(sourcePathRoot, {+ contentDir,+ contentFilter,+ docNameForPath: (relPath) => docNameForFileOperationPath(contentDir, relPath),+ });function listRenamedAssetsForFolderMove(sourcePathRoot: string,@@ -9092,6 +8654,9 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'test-rescan-files', method: 'POST', skipBodyParse: true },);+ const versionOpsService = createVersionOpsService({ getCurrentBranch, contentRoot });+ const skillPlacementOps = createSkillPlacementOpsService();+const handleSaveVersion = withValidation(SaveVersionRequestSchema,async (_req, res, body) => {@@ -9127,13 +8692,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// Parse optional writers from already-validated body.const SAFE_ID_RE = /^[a-zA-Z0-9_-]+$/;let writers: WriterIdentity[] = [];- // True only on the empty-body button path, where `enumerateWipChains`- // already surfaces EVERY WIP chain — upstream included. That makes the- // enumerated set the complete fold list, so saveVersion must not re-append- // the upstream writer (a second rev-parse + a no-op delete on the- // already-reset ref). The explicit-writers and explicit-agentId paths do- // not enumerate upstream and keep the default (fold it).- let foldEnumeratedAll = false;if (Array.isArray(body.writers)) {try {@@ -9160,61 +8718,24 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}}- // Active branch: the button consolidates the branch the user- // is on, not a hardcoded 'main'.- const saveVersionBranch = getCurrentBranch?.() ?? 'main';-- if (writers.length === 0) {- if (svRawAgentId !== undefined) {- // Explicit agentId path (MCP checkpoint tool) — scoped to that agent.- const displayName = svClientName ? `${svAgentName} (${svClientName})` : svAgentName;- writers = [- { id: svAgentId, name: displayName, email: `${svAgentId}@openknowledge.local` },- ];- } else {- // A true empty-body Save Version (the UI button) consolidates ALL- // non-park WIP chains on the active branch — agent + principal +- // classified — so the button matches the user's "group everything I've- // done into a version" mental model. Park-tipped refs hold- // branch-switch state and are excluded. Falls back to the service- // writer when there is no WIP activity at all (an empty checkpoint).- const chains = await enumerateWipChains(shadow, saveVersionBranch);- const foldable = chains.filter((c) => !c.isPark);- writers =- foldable.length > 0- ? foldable.map((c) => ({- id: c.writerId,- name: c.writerId,- email: `${c.writerId}@openknowledge.local`,- }))- : [SERVICE_WRITER];- foldEnumeratedAll = true;- }- }-- const resolvedContentRoot = contentRoot ?? '.';+ // Agent-scoped writer (MCP checkpoint tool path); the service applies+ // the explicit > agent-scoped > fold-everything precedence.+ const agentWriter =+ svRawAgentId !== undefined+ ? {+ id: svAgentId,+ name: svClientName ? `${svAgentName} (${svClientName})` : svAgentName,+ email: `${svAgentId}@openknowledge.local`,+ }+ : undefined;const checkpointSummary = normalizeSummary(typeof body.summary === 'string' ? body.summary : undefined,);- const result = await saveVersion(- shadow,- resolvedContentRoot,- writers,- saveVersionBranch,- checkpointSummary.kind === 'value' ? checkpointSummary.value : undefined,- foldEnumeratedAll ? { includeUpstream: false } : undefined,- );-- getLogger('history').info({ checkpointRef: result.checkpointRef }, 'checkpoint');-- // Rename-log GC trigger: saveVersion deletes WIP refs, which is the- // largest entry-death cliff. Run reachability sweep (no rebuild —- // boot already covered that).- try {- await gcRenameLog(shadow, getOrLoadRenameLogIndex(shadow.gitDir));- } catch (err) {- log.warn({ err }, '[rename-log] post-saveVersion GC failed');- }+ const result = await versionOpsService.saveCheckpoint(shadow, {+ explicitWriters: writers,+ agentWriter,+ summary: checkpointSummary.kind === 'value' ? checkpointSummary.value : undefined,+ });successResponse(res,@@ -10428,108 +9949,48 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);}+ const assetService = createAssetService({+ contentDir,+ // `isPathIgnored` rather than `isExcluded` so the sibling-asset heuristic+ // does not reject legitimate cross-directory references. Exclusion is+ // reported as not-found: the wire shape stays identical to a missing file.+ isPathIgnored: (relativePath) => contentFilter?.isPathIgnored(relativePath) ?? false,+ getAttachmentFolderPath,+ });+ const ASSET_SERVE_ERRORS = {+ 'missing-path': [400, 'urn:ok:error:invalid-request', 'Missing asset path.'],+ 'unsupported-type': [415, 'urn:ok:error:unsupported-asset-type', 'Unsupported asset type.'],+ 'not-found': [404, 'urn:ok:error:asset-not-found', 'Asset not found.'],+ 'invalid-path': [400, 'urn:ok:error:invalid-request', 'Invalid asset path.'],+ } as const;+const handleAsset = withValidation(EmptyRequestSchema,async (req, res) => {try {const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);const assetPath = url.searchParams.get('path');- if (!assetPath || assetPath.includes('\0')) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Missing asset path.', {- handler: 'asset',- });- return;- }- const contentType = assetContentTypeForPath(assetPath);- const assetExt = extname(assetPath).slice(1).toLowerCase();- if (!contentType || !ASSET_EXTENSIONS.has(assetExt)) {- errorResponse(- res,- 415,- 'urn:ok:error:unsupported-asset-type',- 'Unsupported asset type.',- { handler: 'asset' },- );- return;- }- const resolvedContentDir = realpathSync(contentDir);- const requestedPath = resolve(resolvedContentDir, assetPath);- let canonicalPath: string;- try {- canonicalPath = realpathSync(requestedPath);- } catch {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {- handler: 'asset',- });- return;- }- if (!isWithinContentDir(canonicalPath, resolvedContentDir)) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid asset path.', {- handler: 'asset',- });- return;- }- let stat: ReturnType<typeof statSync>;- try {- stat = statSync(canonicalPath);- } catch {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {- handler: 'asset',- });- return;- }- if (!stat.isFile()) {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {- handler: 'asset',- });- return;- }- const relativePath = toContentRelativePath(resolvedContentDir, canonicalPath);- if (relativePath !== assetPath.split('\\').join('/')) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid asset path.', {- handler: 'asset',- });- return;- }- // Direct asset fetches honor the same `.gitignore` / `.okignore`- // exclusions the `createAssetServeMiddleware` admission path- // applies — without this, ignore patterns hide assets from the- // sidebar and SPA but still let `/api/asset?path=...` serve them.- // 404 (not 403) so the wire shape is identical to "missing file"- // — exclusion is opaque.- //- // Use `isPathIgnored` rather than `isExcluded` so the sibling-asset- // heuristic does not reject legitimate cross-directory references- // (e.g., `docs/media/diagram.png` referenced from `docs/guide.md`- // when `docs/media/` has no sibling `.md` of its own).- if (contentFilter?.isPathIgnored(relativePath)) {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {+ const resolution = assetService.resolveServableAsset(assetPath);+ if (!resolution.ok) {+ const [status, type, title] = ASSET_SERVE_ERRORS[resolution.reason];+ errorResponse(res, status, type, title, {handler: 'asset',+ ...(resolution.cause !== undefined ? { cause: resolution.cause } : {}),});return;}- // `html`/`htm` render inline ONLY inside the sandbox CSP below — they- // are intentionally absent from INLINE_RENDERABLE_EXTENSIONS so no other- // branch serves them as a plain same-origin document.- const isSandboxedHtml = SANDBOXED_HTML_EXTENSIONS.has(assetExt);+ const { asset } = resolution;const headers: Record<string, string> = {- 'Content-Type': contentType,- 'Content-Length': String(stat.size),+ 'Content-Type': asset.contentType,+ 'Content-Length': String(asset.size),'X-Content-Type-Options': 'nosniff',- 'Content-Disposition':- INLINE_RENDERABLE_EXTENSIONS.has(assetExt) || isSandboxedHtml ? 'inline' : 'attachment',+ 'Content-Disposition': asset.disposition,'Cache-Control': 'no-store',};- if (assetExt === 'svg') {- headers['Content-Security-Policy'] =- "sandbox; default-src 'none'; style-src 'unsafe-inline'";- } else if (isSandboxedHtml) {- // Opaque sandboxed origin (scripts run, no OK cookies/storage) +- // `connect-src 'none'` so the document can't reach OK's loopback API- // (which allowlists the sandboxed `Origin: null`) or exfiltrate. See- // `SANDBOXED_HTML_CSP`. Mirrors the serve middleware.- headers['Content-Security-Policy'] = SANDBOXED_HTML_CSP;+ if (asset.csp !== null) {+ headers['Content-Security-Policy'] = asset.csp;}+ const canonicalPath = asset.canonicalPath;res.writeHead(200, headers);try {await pipeline(createReadStream(canonicalPath), res);@@ -10600,54 +10061,16 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {try {const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);const assetPath = url.searchParams.get('path');- if (!assetPath || assetPath.includes('\0')) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Missing asset path.', {- handler: 'asset-text',- });- return;- }- const resolvedContentDir = realpathSync(contentDir);- const requestedPath = resolve(resolvedContentDir, assetPath);- let canonicalPath: string;- try {- canonicalPath = realpathSync(requestedPath);- } catch (e) {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {- handler: 'asset-text',- cause: e,- });- return;- }- if (!isWithinContentDir(canonicalPath, resolvedContentDir)) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid asset path.', {- handler: 'asset-text',- });- return;- }- let fileStat: ReturnType<typeof statSync>;- try {- fileStat = statSync(canonicalPath);- } catch (e) {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {- handler: 'asset-text',- cause: e,- });- return;- }- if (!fileStat.isFile()) {- errorResponse(res, 404, 'urn:ok:error:asset-not-found', 'Asset not found.', {- handler: 'asset-text',- });- return;- }- const relativePath = toContentRelativePath(resolvedContentDir, canonicalPath);- if (relativePath !== assetPath.split('\\').join('/')) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid asset path.', {+ const resolution = assetService.resolveTextAsset(assetPath);+ if (!resolution.ok) {+ const [status, type, title] = ASSET_SERVE_ERRORS[resolution.reason];+ errorResponse(res, status, type, title, {handler: 'asset-text',+ ...(resolution.cause !== undefined ? { cause: resolution.cause } : {}),});return;}- if (fileStat.size > TEXT_VIEW_MAX_BYTES) {+ if (resolution.size > TEXT_VIEW_MAX_BYTES) {errorResponse(res,413,@@ -10657,7 +10080,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- const bytes = await readFile(canonicalPath);+ const bytes = await readFile(resolution.canonicalPath);const text = bytes.toString('utf-8');res.writeHead(200, {'Content-Type': 'text/plain; charset=utf-8',@@ -10907,34 +10330,22 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {templateScopeForLog = matched.scope;}- mkdirSync(dirname(fullPath), { recursive: true });- try {- writeFileSync(fullPath, initialContent, { encoding: 'utf-8', flag: 'wx' });- } catch (err) {- if (isAlreadyExistsError(err)) {- errorResponse(res, 409, 'urn:ok:error:doc-already-exists', 'File already exists.', {- handler: 'create-page',- cause: err,- });- return;- }- throw err;- }const docName = stripDocExtension(filePath);- // Eager invalidation: legitimate recreation at a recently-renamed or- // recently-deleted name drops the stale cache entry so the next- // connection admits cleanly. No-op when absent.- recentlyRemovedDocs?.delete(docName);- // Synchronously bump the content filter's sibling-asset dirCount so any- // sibling asset drop that follows is admitted by the `LINKABLE_ASSET_EXTENSIONS`- // rule. The file watcher's `create` event will also increment later,- // which would double-count — so we also `registerWrite` to mark this- // as a self-write, and the watcher skips its own `incrementMdDir` on- // self-writes. See file-watcher.ts for the paired logic.- if (contentFilter) {- contentFilter.incrementMdDir(dirname(docName));+ // Synchronous through recordContributor below: an async yield between+ // the write and the contributor recording lets a pending shadow-commit+ // timer drain the accumulator without this file's attribution.+ const createOutcome = fileOpsService.createPage({+ fullPath,+ docName,+ initialContent,+ });+ if (!createOutcome.ok) {+ errorResponse(res, 409, 'urn:ok:error:doc-already-exists', 'File already exists.', {+ handler: 'create-page',+ cause: createOutcome.cause,+ });+ return;}- registerWrite(fullPath, contentHash(initialContent));switch (actor.kind) {case 'agent':case 'principal':@@ -10957,7 +10368,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);}}- mutateFileIndex?.({ kind: 'create', path: fullPath, docName, content: initialContent });await recordDerivedDocumentBestEffort(docName, initialContent, 'create-page');signalChannel?.('files');if (templateScopeForLog !== undefined) {@@ -10984,6 +10394,57 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'create-page', method: 'POST' },);+ const fileOpsService = createFileOpsService({+ contentDir,+ resolveContentEntryPath,+ docNameForPath: (relPath) => docNameForFileOperationPath(contentDir, relPath),+ docNameToRelativePath,+ listManagedDocNamesUnderFolder: (absFolderPath) =>+ listManagedDocNamesUnderFolderFromDisk(absFolderPath),+ listAffectedDocNames: (index, kind, path) =>+ listAffectedDocNames(index as Map<string, FileIndexEntry>, kind, path),+ getFileIndex,+ getConflictedFiles: () =>+ new Set(+ getSyncEngine?.()+ ?.getConflicts()+ .map((c) => c.file) ?? [],+ ),+ isDocNameInLifecycleConflict: (docName) => {+ const doc = hocuspocus.documents.get(docName);+ return doc !== undefined && isDocInConflict(doc);+ },+ captureAndCloseDocuments,+ markRecentlyRemoved: recentlyRemovedDocs+ ? (docName) => recentlyRemovedDocs.setDeleted(docName)+ : undefined,+ mutateFileIndexDelete: mutateFileIndex+ ? ({ path, docName }) => mutateFileIndex({ kind: 'delete', path, docName })+ : undefined,+ removeFolderIndexEntries,+ upsertFolderIndexPathSegments,+ deleteDerivedDocumentsBestEffort,+ invalidateReferencedAssetsCache,+ signalFiles: () => signalChannel?.('files'),+ nextAvailableDuplicateDocName: (sourceDocName) =>+ nextAvailableDuplicateDocName(contentDir, sourceDocName),+ nextAvailableDuplicateFolderPath: (sourceFolderPath) =>+ nextAvailableDuplicateFolderPath(contentDir, sourceFolderPath),+ resolveDuplicateDocPath: (docName, extension) =>+ resolveDuplicateDocPath(contentDir, docName, extension),+ collectMarkdownCopies: (folderPath) => collectMarkdownCopies(contentDir, folderPath),+ collectFolderPaths: (folderPath) => collectFolderPaths(contentDir, folderPath),+ contentFilter: contentFilter ?? undefined,+ unmarkRecentlyRemoved: recentlyRemovedDocs+ ? (docName) => recentlyRemovedDocs.delete(docName)+ : undefined,+ mutateFileIndexCreate: mutateFileIndex+ ? ({ path, docName, content }) => mutateFileIndex({ kind: 'create', path, docName, content })+ : undefined,+ recordDerivedDocumentBestEffort,+ recordDerivedMutationsBestEffort,+ });+const handleCreateFolder = withValidation(CreateFolderRequestSchema,async (_req, res, body) => {@@ -11028,17 +10489,13 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- const fullPath = resolveContentEntryPath(contentDir, 'folder', folderPath);- if (existsSync(fullPath)) {+ const outcome = fileOpsService.createFolder(folderPath);+ if (!outcome.ok) {errorResponse(res, 409, 'urn:ok:error:doc-already-exists', 'Folder already exists.', {handler: 'create-folder',});return;}-- tracedMkdirSync(fullPath, { recursive: true });- upsertFolderIndexPathSegments(folderPath);- signalChannel?.('files');successResponse(res,200,@@ -11100,11 +10557,16 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {if (kind === 'file') {probeAndRegisterSourceFileExtension(contentDir, requestedPath);}- const sourcePath = resolveContentEntryPath(contentDir, kind, requestedPath);- if (!existsSync(sourcePath)) {- if (kind === 'file') {- const folderSourcePath = resolveContentEntryPath(contentDir, 'folder', requestedPath);- if (existsSync(folderSourcePath) && statSync(folderSourcePath).isDirectory()) {++ const outcome = await fileOpsService.duplicatePath(kind, requestedPath, requestedDocName);+ if (!outcome.ok) {+ switch (outcome.kind) {+ case 'not-found':+ errorResponse(res, 404, 'urn:ok:error:doc-not-found', `${kind} does not exist.`, {+ handler: 'duplicate-path',+ });+ return;+ case 'type-mismatch':errorResponse(res,400,@@ -11113,269 +10575,42 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'duplicate-path' },);return;- }- }- errorResponse(res, 404, 'urn:ok:error:doc-not-found', `${kind} does not exist.`, {- handler: 'duplicate-path',- });- return;- }- const sourceStat = statSync(sourcePath);- if (- (kind === 'file' && !sourceStat.isFile()) ||- (kind === 'folder' && !sourceStat.isDirectory())- ) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', `Target path is not a ${kind}.`, {- handler: 'duplicate-path',- });- return;- }-- // Conflict-aware refusal. Duplicating a conflicted source would- // copy the raw `<<<<<<< HEAD` / `=======` / `>>>>>>>` marker bytes- // from disk into a new file at the destination, producing a broken- // duplicate. Refuse with 409; the user must resolve the conflict- // first. Dual-source check mirrors handleRenamePath /- // handleDeletePath — `hocuspocus.documents.get()` returns undefined- // for evicted docs; the ConflictStore fallback catches that case.- // Enumerate from disk (not the lagging file index) so the conflict- // gate sees every on-disk child of the folder — the chokidar watcher- // populates the index asynchronously, so right after a fresh- // `write` create the index lags and a conflicted child would- // be silently skipped, copying its marker bytes intact. Same root- // cause and fix as handleRenamePath's pre-check. Also registers- // each child's on-disk extension for extension-less legacy docNames.- const duplicateSourceDocNames =- kind === 'file'- ? [docNameForFileOperationPath(contentDir, requestedPath)]- : listManagedDocNamesUnderFolderFromDisk(- resolveContentEntryPath(contentDir, 'folder', requestedPath),+ case 'conflict':+ respondDocInConflict(+ res,+ new DocInConflictError({ file: outcome.file }),+ 'duplicate-path',);- const duplicateEngine = getSyncEngine?.();- const duplicateTrackedFiles = new Set(- duplicateEngine ? duplicateEngine.getConflicts().map((c) => c.file) : [],- );- for (const affected of duplicateSourceDocNames) {- const affectedDocName = affected;- const doc = hocuspocus.documents.get(affectedDocName);- const filePath = docNameToRelativePath(affectedDocName);- const conflictedByLifecycle = doc !== undefined && isDocInConflict(doc);- const conflictedByStore = duplicateTrackedFiles.has(filePath);- if (conflictedByLifecycle || conflictedByStore) {- respondDocInConflict(res, new DocInConflictError({ file: filePath }), 'duplicate-path');- return;- }- }-- let duplicatedPath: string;- let duplicatedDocNames: string[] = [];-- if (kind === 'file') {- const sourceExtension = extname(sourcePath);- const next = nextAvailableDuplicateDocName(contentDir, requestedDocName);- duplicatedPath = next.docName;- if (- isSystemDoc(duplicatedPath) ||- isConfigDoc(duplicatedPath) ||- contentFilter?.isExcluded(`${duplicatedPath}${sourceExtension}`)- ) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- 'Duplicated document destination is excluded by the project content config.',- { handler: 'duplicate-path' },- );- return;- }- const destinationPath = resolveDuplicateDocPath(- contentDir,- duplicatedPath,- sourceExtension,- );- const content = readFileSync(sourcePath, 'utf-8');- const destinationDir = dirname(destinationPath);- const destinationDirExisted = existsSync(destinationDir);- try {- tracedMkdirSync(destinationDir, { recursive: true });- tracedWriteFileSync(destinationPath, content, { encoding: 'utf-8', flag: 'wx' });- } catch (err) {- if (isAlreadyExistsError(err)) {+ return;+ case 'destination-excluded':errorResponse(res,- 409,- 'urn:ok:error:doc-already-exists',- 'A file at the duplicate destination already exists.',- { handler: 'duplicate-path', cause: err },+ 400,+ 'urn:ok:error:invalid-request',+ kind === 'file'+ ? 'Duplicated document destination is excluded by the project content config.'+ : 'Duplicated folder destination is excluded by the project content config.',+ { handler: 'duplicate-path' },);return;- }- if (!destinationDirExisted) {- try {- tracedRmdirSync(destinationDir);- } catch (cleanupErr) {- const cleanupCode = errnoCode(cleanupErr);- if (cleanupCode !== 'ENOENT' && cleanupCode !== 'ENOTEMPTY') {- log.warn(- { destinationDir, err: cleanupErr },- '[duplicate-path] failed to clean duplicate parent directory',- );- }- }- }- throw err;- }- let didIncrementMdDir = false;- try {- registerDocExtension(duplicatedPath, sourceExtension);- recentlyRemovedDocs?.delete(duplicatedPath);- if (contentFilter) {- contentFilter.incrementMdDir(dirname(duplicatedPath));- didIncrementMdDir = true;- }- registerWrite(destinationPath, contentHash(content));- mutateFileIndex?.({- kind: 'create',- path: destinationPath,- docName: duplicatedPath,- content,- });- duplicatedDocNames = [duplicatedPath];- } catch (err) {- try {- tracedRmSync(destinationPath, { force: true });- } catch (cleanupErr) {- log.warn(- { destinationPath, err: cleanupErr },- '[duplicate-path] failed to clean partial file duplicate',- );- }- forgetDocExtension(duplicatedPath);- if (contentFilter && didIncrementMdDir) {- contentFilter.decrementMdDir(dirname(duplicatedPath));- }- mutateFileIndex?.({- kind: 'delete',- path: destinationPath,- docName: duplicatedPath,- });- throw err;- }- await recordDerivedDocumentBestEffort(duplicatedPath, content, 'duplicate-path-file');- } else {- const next = nextAvailableDuplicateFolderPath(contentDir, requestedPath);- duplicatedPath = next.folderPath;- if (contentFilter?.isDirExcluded(duplicatedPath)) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- 'Duplicated folder destination is excluded by the project content config.',- { handler: 'duplicate-path' },- );- return;- }- const destinationPath = resolveContentEntryPath(contentDir, 'folder', duplicatedPath);- const copiedDocRollbackLedger: Array<{- docName: string;- fullPath: string;- extensionRegistered: boolean;- dirCountIncremented: boolean;- fileIndexRegistered: boolean;- }> = [];- try {- tracedCpSync(sourcePath, destinationPath, {- recursive: true,- errorOnExist: true,- force: false,- });- } catch (err) {- if (isAlreadyExistsError(err)) {+ case 'already-exists':errorResponse(res,409,'urn:ok:error:doc-already-exists',- 'A folder at the duplicate destination already exists.',- { handler: 'duplicate-path', cause: err },+ `A ${kind} at the duplicate destination already exists.`,+ { handler: 'duplicate-path', cause: outcome.cause },);return;- }- throw err;- }- const derivedMutations: DerivedDocumentIndexMutation[] = [];- try {- for (const folderPath of collectFolderPaths(contentDir, duplicatedPath)) {- upsertFolderIndexPathSegments(folderPath);- }- const copiedDocs = collectMarkdownCopies(contentDir, duplicatedPath);- duplicatedDocNames = copiedDocs.map((doc) => doc.docName);- for (const doc of copiedDocs) {- const rollbackEntry = {- docName: doc.docName,- fullPath: doc.fullPath,- extensionRegistered: false,- dirCountIncremented: false,- fileIndexRegistered: false,- };- copiedDocRollbackLedger.push(rollbackEntry);- const sourceExtension = extname(doc.fullPath);- registerDocExtension(stripDocExtension(doc.docName), sourceExtension);- rollbackEntry.extensionRegistered = true;- recentlyRemovedDocs?.delete(doc.docName);- if (contentFilter) {- contentFilter.incrementMdDir(dirname(doc.docName));- rollbackEntry.dirCountIncremented = true;- }- registerWrite(doc.fullPath, contentHash(doc.content));- rollbackEntry.fileIndexRegistered = true;- mutateFileIndex?.({- kind: 'create',- path: doc.fullPath,- docName: doc.docName,- content: doc.content,- });- derivedMutations.push({- kind: 'upsert',- documentName: doc.docName,- markdown: doc.content,- });- }- } catch (err) {- for (const rollbackEntry of copiedDocRollbackLedger.reverse()) {- if (rollbackEntry.fileIndexRegistered) {- try {- mutateFileIndex?.({- kind: 'delete',- path: rollbackEntry.fullPath,- docName: rollbackEntry.docName,- });- } catch (rollbackErr) {- log.warn(- { docName: rollbackEntry.docName, err: rollbackErr },- '[duplicate-path] failed to roll back copied file-index row',- );- }- }- if (rollbackEntry.dirCountIncremented) {- contentFilter?.decrementMdDir(dirname(rollbackEntry.docName));- }- if (rollbackEntry.extensionRegistered) {- forgetDocExtension(stripDocExtension(rollbackEntry.docName));- }- }- removeFolderIndexEntries(duplicatedPath);- try {- tracedRmSync(destinationPath, { recursive: true, force: true });- } catch (cleanupErr) {- log.warn(- { destinationPath, err: cleanupErr },- '[duplicate-path] failed to clean partial folder duplicate',+ default: {+ const _exhaustive: never = outcome;+ throw new Error(+ `Unhandled duplicate outcome: ${String((_exhaustive as { kind?: unknown }).kind)}`,);}- throw err;}- await recordDerivedMutationsBestEffort(derivedMutations, 'duplicate-path-folder');}+ const { duplicatedPath, duplicatedDocNames } = outcome;switch (actor.kind) {case 'agent':@@ -11886,137 +11121,38 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- const targetPath =- operationKind === 'asset'- ? resolveContentEntryPath(contentDir, 'folder', operationPath)- : resolveContentEntryPath(contentDir, operationKind, operationPath);- if (!existsSync(targetPath)) {- errorResponse(- res,- 404,- 'urn:ok:error:doc-not-found',- `${operationKind} does not exist.`,- {- handler: 'delete-path',- },- );- return;- }-- const targetStat = statSync(targetPath);- if (- (operationKind === 'file' && !targetStat.isFile()) ||- (operationKind === 'asset' && !targetStat.isFile()) ||- (operationKind === 'folder' && !targetStat.isDirectory())- ) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- `Target path is not a ${operationKind}.`,- { handler: 'delete-path' },- );- return;- }-- // Enumerate descendants from disk (not the lagging file index) so the- // folder delete sees every on-disk child. The chokidar watcher- // populates the index asynchronously, so right after a fresh- // `write` create it lags on-disk truth: reading it here would- // return an empty `deletedDocNames`, so `captureAndCloseDocuments` and- // the `recentlyRemovedDocs` population below would be skipped while- // `tracedRmSync` still removes the directory — orphaning the in-memory- // Y.Docs (silent data loss). Same root cause and fix as- // handleRenamePath's affected-docs scan. The walk runs before the disk- // delete, so disk is authoritative here; it also registers each- // child's on-disk extension for extension-less legacy docNames.- const deletedDocNames =- operationKind === 'asset'- ? []- : operationKind === 'file'- ? [docNameForFileOperationPath(contentDir, operationPath)]- : listManagedDocNamesUnderFolderFromDisk(- resolveContentEntryPath(contentDir, 'folder', operationPath),- );-- // Conflict-aware refusal. Deleting a conflicted doc would- // discard the in-flight resolution state; resolution must complete- // first via `resolve_conflict` (or be aborted via `git merge --abort`- // per the documented recovery procedure). Scan every affected- // doc — for folder deletes, ANY conflicted child blocks the operation.- //- // Dual-source check (same rationale as handleRenamePath above):- // hocuspocus.documents.get() returns undefined for docs evicted- // from memory; ConflictStore catches that eviction race.- const deleteEngine = getSyncEngine?.();- const deleteTrackedFiles = new Set(- deleteEngine ? deleteEngine.getConflicts().map((c) => c.file) : [],- );- for (const affected of deletedDocNames) {- const affectedDocName = affected;- const doc = hocuspocus.documents.get(affectedDocName);- const filePath = docNameToRelativePath(affectedDocName);- const conflictedByLifecycle = doc !== undefined && isDocInConflict(doc);- const conflictedByStore = deleteTrackedFiles.has(filePath);- if (conflictedByLifecycle || conflictedByStore) {- respondDocInConflict(res, new DocInConflictError({ file: filePath }), 'delete-path');- return;- }- }-- await captureAndCloseDocuments(deletedDocNames, 'deleted-upstream');-- // Populate the per-process LRU cache BEFORE the disk delete so any- // connection that observes the file gone via the watcher also sees the- // cache entry — closes the race where a fast reconnect could land- // between the unlink and the cache write. Filter via the standard- // `isSystemDoc()`/`isConfigDoc()` STOP gate; synthetic docs cannot- // appear in `deletedDocNames` today (path validation rejects them),- // but the filter stays defense-in-depth.- if (recentlyRemovedDocs) {- for (const docName of deletedDocNames) {- if (isSystemDoc(docName) || isConfigDoc(docName)) continue;- recentlyRemovedDocs.setDeleted(docName);- console.info(- JSON.stringify({- event: 'recently-removed-docs-populate',- docName,- kind: 'deleted',- source: 'handleDeletePath',- }),+ const outcome = await fileOpsService.deletePath(operationKind, operationPath);+ if (!outcome.ok) {+ if (outcome.kind === 'not-found') {+ errorResponse(+ res,+ 404,+ 'urn:ok:error:doc-not-found',+ `${operationKind} does not exist.`,+ { handler: 'delete-path' },+ );+ } else if (outcome.kind === 'type-mismatch') {+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ `Target path is not a ${operationKind}.`,+ { handler: 'delete-path' },+ );+ } else {+ respondDocInConflict(+ res,+ new DocInConflictError({ file: outcome.file }),+ 'delete-path',);}+ return;}-- if (operationKind === 'file' || operationKind === 'asset') {- tracedUnlinkSync(targetPath);- } else {- tracedRmSync(targetPath, { recursive: true, force: false });- removeFolderIndexEntries(operationPath);- }- invalidateReferencedAssetsCache();-- // Refresh the file index so subsequent doc-list reads don't include- // the just-deleted entries. Watcher events would eventually do this- // anyway but the doc-list response needs to be consistent right now.- // Routes through the typed `mutateFileIndex` accessor (live map);- // the pre-PR pattern of `getFileIndex() + as Map cast` silently- // dead-ended once `getFileIndex()` flipped to returning a snapshot.- for (const docName of deletedDocNames) {- mutateFileIndex?.({- kind: 'delete',- path: resolve(contentDir, docNameToRelativePath(docName)),- docName,- });- }- await deleteDerivedDocumentsBestEffort(deletedDocNames, 'delete-path');-- signalChannel?.('files');successResponse(res,200,DeletePathSuccessSchema,- { deletedDocNames },+ { deletedDocNames: outcome.deletedDocNames },{ handler: 'delete-path' },);} catch (e) {@@ -12103,88 +11239,12 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- if (operationKind === 'asset') {- invalidateReferencedAssetsCache();- signalChannel?.('files');- successResponse(- res,- 200,- TrashCleanupSuccessSchema,- { deletedDocNames: [] },- { handler: 'trash-cleanup' },- );- return;- }-- // Source of truth for "what to purge" is the in-memory fileIndex.- // The OS-level move-to-Trash happened in Step 1; the watcher MAY- // have processed it already (then the index is empty for this- // path), or NOT (then the index still holds the entries). For- // the idempotent fast-path: when the index lacks the entries,- // return 200 + empty array — the desired end state (gone) is- // already true; nothing left for us to do.- const initialIndex = getFileIndex();- const deletedDocNames =- operationKind === 'file'- ? initialIndex.has(operationDocName)- ? [operationDocName]- : []- : listAffectedDocNames(initialIndex, operationKind, path);-- invalidateReferencedAssetsCache();-- if (deletedDocNames.length === 0) {- successResponse(- res,- 200,- TrashCleanupSuccessSchema,- { deletedDocNames: [] },- { handler: 'trash-cleanup' },- );- return;- }-- await captureAndCloseDocuments(deletedDocNames, 'deleted-upstream');-- if (recentlyRemovedDocs) {- for (const docName of deletedDocNames) {- if (isSystemDoc(docName) || isConfigDoc(docName)) continue;- recentlyRemovedDocs.setDeleted(docName);- console.info(- JSON.stringify({- event: 'recently-removed-docs-populate',- docName,- kind: 'deleted',- source: 'handleTrashCleanup',- }),- );- }- }-- // Synchronously purge the in-memory index so subsequent doc-list- // reads return the post-trash state immediately. The file-watcher- // will also process the OS-level deletion event eventually; both- // pathways converge on the same end state. Routes through the- // typed `mutateFileIndex` accessor (live map) — the pre-PR- // `getFileIndex() + as Map cast` pattern silently dead-ended- // once `getFileIndex()` flipped to returning a snapshot.- for (const docName of deletedDocNames) {- mutateFileIndex?.({- kind: 'delete',- path: resolve(contentDir, docNameToRelativePath(docName)),- docName,- });- }- if (operationKind === 'folder') {- removeFolderIndexEntries(path);- }- await deleteDerivedDocumentsBestEffort(deletedDocNames, 'trash-cleanup');-- // Synchronous CC1 emit closes the race where the renderer expects- // the updated tree right after the response. The watcher's later- // emit is idempotent at the consumer (per-channel seq dedup).- signalChannel?.('files');-+ const { deletedDocNames } = await fileOpsService.trashCleanup(+ operationKind,+ path,+ operationDocName,+ 'handleTrashCleanup',+ );successResponse(res,200,@@ -12424,272 +11484,116 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return;}- const resolvedContentDir = resolve(contentDir);- let rawAttachmentFolderPath: string;- try {- rawAttachmentFolderPath =- placement === 'parent-dir'- ? DEFAULT_ATTACHMENT_FOLDER_PATH- : (getAttachmentFolderPath?.() ?? DEFAULT_ATTACHMENT_FOLDER_PATH);- } catch (err) {- cleanupTempfile();- log.error({ err }, '[upload] project config has invalid content.attachmentFolderPath');- errorResponse(- res,- 500,- 'urn:ok:error:internal-server-error',- 'Server configuration error: invalid attachment folder path.',+ const outcome = await assetService.storeUpload({+ tempPath,+ sha,+ byteLength,+ filename,+ parentDocName,+ placement,+ });+ if (outcome.ok) {+ log.info({- handler: 'upload-asset',- cause: err,+ event: 'upload',+ endpoint: req.url ?? '/api/upload',+ agentId,+ agentName,+ dedup: outcome.deduped,+ mime: outcome.mime,+ size: byteLength,+ // `destPath` is the contentDir-relative asset path. High-+ // cardinality by nature — fine as a log field consumed by text-+ // search / by-incident filtering; NEVER promote it to a metric+ // label (per-asset label explosion).+ destPath: outcome.path,+ httpStatus: 200,},+ outcome.deduped ? '[upload] dedup hit' : '[upload] write ok',+ );+ // RFC 9457 §3 success path: drop the `ok: true` wrapper. Wire shape+ // is `{ src, path, deduped }`; clients discriminate on HTTP status.+ successResponse(+ res,+ 200,+ UploadAssetSuccessSchema,+ { src: outcome.src, path: outcome.path, deduped: outcome.deduped },+ { handler: 'upload-asset' },);return;}- if (!isValidAttachmentFolderPath(rawAttachmentFolderPath)) {- cleanupTempfile();- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid attachment folder path.', {- handler: 'upload-asset',- });- return;- }- const attachmentFolderPath = normalizeAttachmentFolderPath(rawAttachmentFolderPath);- const destDir = resolveUploadDestDir(parentDocName, attachmentFolderPath, resolvedContentDir);- if (!isWithinContentDir(destDir, resolvedContentDir)) {- cleanupTempfile();- errorResponse(res, 400, 'urn:ok:error:path-escape', 'Path escape detected.', {- handler: 'upload-asset',- });- return;- }- // Pre-mkdir symlink-escape check: walks up from destDir to the- // deepest existing ancestor and rejects if its realpath escapes contentDir.- // Doing this before `mkdirSync({ recursive: true })` prevents mkdir from- // following a parent symlink and materializing a fresh directory outside- // contentDir. The post-mkdir realpath check below remains as defense-in-- // depth against TOCTOU symlink-replace races between this check and mkdir.- try {- assertNoSymlinkEscape(destDir, resolvedContentDir);- } catch (err) {- cleanupTempfile();- const message = err instanceof Error ? err.message : String(err);- if (message.startsWith('symlink-escape:')) {- errorResponse(res, 400, 'urn:ok:error:path-escape', 'Path escape detected.', {- handler: 'upload-asset',- });+ switch (outcome.kind) {+ case 'config-error':+ log.error(+ { err: outcome.cause },+ '[upload] project config has invalid content.attachmentFolderPath',+ );+ errorResponse(+ res,+ 500,+ 'urn:ok:error:internal-server-error',+ 'Server configuration error: invalid attachment folder path.',+ { handler: 'upload-asset', cause: outcome.cause },+ );return;- }- log.error({ err, destDir }, '[upload] failed to validate destination directory');- errorResponse(res, 500, 'urn:ok:error:storage-error', 'Storage error.', {- handler: 'upload-asset',- cause: err,- });- return;- }- // mkdir -p the destination — bare-name / nested attachmentFolderPath- // values produce directories that may not exist at first upload.- try {- mkdirSync(destDir, { recursive: true });- } catch (err) {- if (!isAlreadyExistsError(err)) {- cleanupTempfile();- // Classify the errno through the same typed table the streaming-- // write path uses so ENOSPC/EDQUOT route through 507 storage-full- // and EROFS/EACCES/EPERM route through 500 storage-readonly —- // SDK consumers branch on the URN, not the errno, so collapsing- // every errno into generic storage-error breaks that contract.- const reason = classifyUploadErrno(err as NodeJS.ErrnoException);- errorResponse(res, uploadStatusFor(reason), reason, uploadTitleFor(reason), {+ case 'invalid-attachment-folder':+ errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid attachment folder path.', {handler: 'upload-asset',- cause: err,- detail: 'failed to create attachment directory',});return;- }- }-- // Symlink escape check: realpath the dest dir and compare against realpath'd contentDir- try {- const realDestDir = realpathSync(destDir);- let realContentDir: string;- try {- realContentDir = realpathSync(resolvedContentDir);- } catch {- realContentDir = resolvedContentDir;- }- if (!isWithinContentDir(realDestDir, realContentDir)) {- cleanupTempfile();+ case 'path-escape':errorResponse(res, 400, 'urn:ok:error:path-escape', 'Path escape detected.', {handler: 'upload-asset',+ ...(outcome.cause === undefined ? {} : { cause: outcome.cause }),});return;- }- } catch (e) {- const code = errnoCode(e);- if (code === 'ENOENT') {- // Directory doesn't exist yet — will be created below; no symlink escape possible- } else {- cleanupTempfile();- errorResponse(res, 400, 'urn:ok:error:path-escape', 'Path escape detected.', {+ case 'dest-validation-error':+ log.error(+ { err: outcome.cause, destDir: outcome.destDir },+ '[upload] failed to validate destination directory',+ );+ errorResponse(res, 500, 'urn:ok:error:storage-error', 'Storage error.', {handler: 'upload-asset',- cause: e,+ cause: outcome.cause,});return;- }- }-- // Accept-all: every file is accepted — there's no user-facing byte cap- // post-streaming (disk fullness surfaces as 507 instead). The magic-- // byte sniff is only consulted to (a) preserve the SVG `<img>`-only- // routing for security and (b) recover an extension when the upload- // arrived with a generic clipboard filename. Non-sniffable bytes are- // accepted under the client-supplied filename.- //- const SNIFF_HEAD_BYTES = 4100;- const head = readTempFileHead(tempPath, SNIFF_HEAD_BYTES);- const fileTypeResult = await fileTypeFromBuffer(head);- let detectedMime: string | undefined = fileTypeResult?.mime;- let detectedExt: string | undefined = fileTypeResult?.ext;- // file-type can't detect SVG (text-based, no magic bytes) — check manually.- // STOP: this fallback is LOAD-BEARING — SVG must render via- // <img>, never inline DOM. Do not remove without a compensating guard.- if (!detectedMime) {- // Strip a leading UTF-8 BOM (U+FEFF) before the pattern match.- // `trimStart()` removes ECMAScript whitespace but not the BOM, so a- // file starting with `\xEF\xBB\xBF<svg ...>` would otherwise evade the- // head check the comment above documents as the SVG-disguised-as-PNG- const headText = head.subarray(0, 256).toString('utf-8').replace(/^/, '').trimStart();- if (- headText.startsWith('<svg') ||- (headText.startsWith('<?xml') && headText.includes('<svg'))- ) {- detectedMime = 'image/svg+xml';- detectedExt = 'svg';- }- }-- // Same-dir sha256 dedup. Bounded scan over destDir, skipped entirely- // when DEFAULT_DEDUP_MODE === 'off'. The dedup test happens BEFORE- // filename synthesis so a duplicate paste preserves the existing- // on-disk basename instead of producing a fresh pasted-<ts>.png stub.- // Server returns { deduped: true } so the client surfaces a toast.- //- // The hash + size come from the streaming pipeline (no buffer). On a- // dedup hit the tempfile is unlinked and we short-circuit without- // touching the destDir inode — `linkTempToFinalWithCollisionRetry`- // never runs.- if (DEFAULT_DEDUP_MODE === 'same-dir') {- const existing = await findDuplicateAsset(destDir, sha, byteLength);- if (existing) {- cleanupTempfile();- const relPath = toPosix(relative(contentDir, resolve(destDir, existing)));- log.info(+ case 'mkdir-failed':+ errorResponse(+ res,+ uploadStatusFor(outcome.reason),+ outcome.reason,+ uploadTitleFor(outcome.reason),+ {+ handler: 'upload-asset',+ cause: outcome.cause,+ detail: 'failed to create attachment directory',+ },+ );+ return;+ case 'write-failed':+ log.error({event: 'upload',endpoint: req.url ?? '/api/upload',+ requestId: getRequestId(req),agentId,agentName,- dedup: true,- mime: detectedMime ?? null,+ filename: outcome.filename,size: byteLength,- destPath: relPath,- httpStatus: 200,+ reason: outcome.reason,+ httpStatus: uploadStatusFor(outcome.reason),+ err: outcome.cause,},- '[upload] dedup hit',+ '[upload] write failed',);- // RFC 9457 §3 success path: drop the `ok: true` wrapper. Wire- // shape is `{ src, path, deduped }` with `Content-Type:- // application/json`. Clients use HTTP-status discrimination- // (`if (!res.ok)`) to choose between this success schema and- // `ProblemDetailsSchema`.- successResponse(+ errorResponse(res,- 200,- UploadAssetSuccessSchema,- { src: existing, path: relPath, deduped: true },- { handler: 'upload-asset' },+ uploadStatusFor(outcome.reason),+ outcome.reason,+ uploadTitleFor(outcome.reason),+ { handler: 'upload-asset', cause: outcome.cause },);return;- }- }-- // GENERIC_PASTE_NAMES: clipboard paste arrives with synthetic names- // ("image.png", "Clipboard 2024-04-21 14:23:45"). Replace with a- // timestamp stem so the disk filename is human-meaningful.- let finalFilename: string;- const isGenericPaste = !filename || filename === 'upload' || GENERIC_PASTE_NAMES.test(filename);- if (isGenericPaste) {- const now = new Date();- const ts = now- .toISOString()- .replace(/[-:T]/g, '')- .slice(0, 14)- .replace(/(\d{8})(\d{6})/, '$1-$2');- // Prefer the sniffed extension when present; otherwise try the- // client-supplied extname, finally fall back to .bin.- const fallbackExt = filename ? extname(filename).slice(1) : '';- const ext = detectedExt ?? fallbackExt ?? '';- finalFilename = ext === '' ? `pasted-${ts}` : `pasted-${ts}.${ext}`;- } else {- finalFilename = sanitizeFilename(filename);- }-- try {- const destFilename = linkTempToFinalWithCollisionRetry(tempPath, destDir, finalFilename);- const relPath = toPosix(relative(contentDir, resolve(destDir, destFilename)));- log.info(- {- event: 'upload',- endpoint: req.url ?? '/api/upload',- agentId,- agentName,- dedup: false,- mime: detectedMime ?? null,- size: byteLength,- // `destPath` is the contentDir-relative asset path. High-- // cardinality by nature — a vault with 10K assets produces- // 10K distinct values. Fine as a log field consumed by text-- // search / by-incident filtering; NEVER promote it to a- // metric label (Prometheus / Datadog will blow up memory on- // per-asset label explosion). Keep the nested-context shape- // below if you later route these through an aggregator so- // auto-label-extraction honors the sub-object convention.- destPath: relPath,- httpStatus: 200,- },- '[upload] write ok',- );- successResponse(- res,- 200,- UploadAssetSuccessSchema,- { src: destFilename, path: relPath, deduped: false },- { handler: 'upload-asset' },- );- } catch (e) {- // linkTempToFinalWithCollisionRetry best-effort unlinks the tempfile- // on throw; no extra cleanupTempfile() call needed here.- const reason: UploadWriteReason =- e instanceof UploadWriteError ? e.reason : 'urn:ok:error:storage-error';- log.error(- {- event: 'upload',- endpoint: req.url ?? '/api/upload',- requestId: getRequestId(req),- agentId,- agentName,- filename: finalFilename,- size: byteLength,- reason,- httpStatus: uploadStatusFor(reason),- err: e,- },- '[upload] write failed',- );- errorResponse(res, uploadStatusFor(reason), reason, uploadTitleFor(reason), {- handler: 'upload-asset',- cause: e,- });}}@@ -16162,6 +15066,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// live at `<home>/.ok/skills/`; the user-level install marker is// `<home>/.ok/local/installed-skills.json` (readInstalledSkills(skillsHome)).const skillsHome = homeDirOverride ?? homedir();+ const skillInstallOps = createSkillInstallOpsService({ contentDir, skillsHome });/*** Resolve a skill scope to its absolute `.ok/skills` store root. Project@@ -16266,21 +15171,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);}- /**- * Whether a placement root is OK's own state area and therefore refused.- *- * `.ok/` holds OK's own state (config, local runtime data, the shadow repo),- * so a skill may not be placed inside it — with one exception that is not a- * special case so much as history: `.ok/skills` is the retired store, and- * projects that predate the retirement still have real skills sitting there.- * Refusing it would strand them, so it stays placeable at BOTH scopes; a- * placement there is an ordinary custom root like any other.- */- function isRefusedOkPlacementRoot(rootRel: string): boolean {- const underOk = rootRel === '.ok' || rootRel.startsWith('.ok/');- return underOk && rootRel !== LEGACY_SKILL_STORE_ROOT;- }-/*** How a skill should be re-projected into its host dirs.*@@ -19142,301 +18032,33 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}}- /**- * What the shared import spine concluded. The spine returns this rather than- * writing the response so the bulk path can run it once per skill against a- * SINGLE clone and report per-skill rows; `respondSkillImport` applies the- * single-skill HTTP shape.- */- type SkillImportOutcome =- | { ok: true; body: z.infer<typeof SkillImportSuccessSchema> }- | {- ok: false;- status: Parameters<typeof errorResponse>[1];- urn: Parameters<typeof errorResponse>[2];- title: string;- detail?: string;- cause?: unknown;- };-- /** Apply a spine outcome to a single-skill response (`/api/skill/import`, upload). */function respondSkillImport(res: ServerResponse, outcome: SkillImportOutcome): void {if (outcome.ok) {successResponse(res, 200, SkillImportSuccessSchema, outcome.body, {handler: 'skill-import',});- return;- }- errorResponse(res, outcome.status, outcome.urn, outcome.title, {- handler: 'skill-import',- ...(outcome.detail !== undefined ? { detail: outcome.detail } : {}),- ...(outcome.cause !== undefined ? { cause: outcome.cause } : {}),- });- }-- // Shared import spine (steps 2-6): given an on-disk skill dir + provenance,- // dedupe by content hash, land under a free `-imported` name on collision,- // write SKILL.md + bundle files via the sanctioned writers, attribute +- // shadow-commit (project scope), record the lockfile entry, and best-effort- // project into editor dirs. Returns an outcome; the caller writes the- // response. Used by `/api/skill/import` (by-reference), `/api/skill-upload`- // (by-bytes), and `/api/skills/import-bulk` (many skills, one clone). The- // CALLER owns temp-dir cleanup (its `finally`), so this never cleans up.- async function runSkillImport(params: {- acquiredDir: string;- scope: Parameters<typeof resolveSkillsRoot>[0];- sourceLabel: string;- ref?: string;- publisher?: string;- upstreamSkill?: string;- actor: Parameters<typeof attributeOkArtifactWrite>[0];- /** Skip the default-editor auto-projection — for callers that install- * explicitly afterwards (explore preview toggles / custom-path place). */- skipProjection?: boolean;- }): Promise<SkillImportOutcome> {- const { acquiredDir, scope, sourceLabel, ref, publisher, upstreamSkill, actor } = params;- if (!projectDir) {- return {- ok: false,- status: 400,- urn: 'urn:ok:error:invalid-request',- title: 'No project root resolved.',- detail: 'NO_PROJECT_ROOT',- };- }- // Pre-flight the caps by stat, BEFORE the parse materializes every byte —- // `importedBundleLimitError` below runs on an already-built array, which a- // repo of half-gigabyte blobs never reaches.- const oversize = acquiredBundleTooLarge(acquiredDir);- if (oversize) {- return {- ok: false,- status: 400,- urn: 'urn:ok:error:invalid-request',- title: 'Skill bundle exceeds import limits.',- detail: oversize,- };- }- const acquired = parseSkillDir(acquiredDir);- if (!acquired) {- return {- ok: false,- status: 422,- urn: 'urn:ok:error:invalid-request',- title: 'Source has no readable skill.',- };- }- const limitError = importedBundleLimitError(acquired);- if (limitError) {- return {- ok: false,- status: 400,- urn: 'urn:ok:error:invalid-request',- title: 'Skill bundle exceeds import limits.',- detail: limitError,- };- }-- // 2. Dedupe by contentHash via the lockfile (re-import is a no-op).- const lockPath = join(scope === 'project' ? projectDir : skillsHome, ...SKILLS_LOCK_REL);- const lock = readSkillsLock(lockPath);- // WRITE-PATH INVERSION (store retirement, step 1): NEW imports land- // IN-PLACE at the vendor-neutral `.agents/skills` hub — versioned, listed,- // and read at their real path from day one. The store gains no new- // residents; existing store bundles keep working until their migration.- const importBase = scope === 'project' ? contentDir : skillsHome;- const importHomeRel = resolveDefaultSkillHomeRel(importBase, scope);- const importRoot = resolve(importBase, importHomeRel);- const dupName = findByContentHash(lock, acquired.contentHash);- if (dupName && resolveSkillDirForRead(scope, dupName) !== null) {- return {- ok: true,- body: {- name: dupName,- path: `${dupName}/SKILL.md`,- created: false,- alreadyImported: true,- provenance: { source: sourceLabel, ref, contentHash: acquired.contentHash, publisher },- warnings: [],- },- };- }-- // 3. Collision with an existing skill → land under a free `-imported` name.- // `acquired.name` is attacker-controlled (imported SKILL.md frontmatter).- // applySkillWrite's validateName blocks a malformed WRITE, but the existence- // probes below resolve the name against skillsRoot — sanitize first so a- // `../…` name can't turn firstFreeSkillName into a filesystem existence- // oracle. No-op for a normal `[a-z0-9-]` name. (defense-in-depth)- const probeName = sanitizeFilename(acquired.name);- // Collision probe spans the WHOLE registry (store + every in-place host- // dir), not just the import root — a same-name skill at `.claude/skills`- // would fork against the new `.agents` bundle.- const nameTaken = (n: string) =>- resolveSkillDirForRead(scope, n) !== null || existsSync(resolve(importRoot, n, 'SKILL.md'));- const collided = nameTaken(probeName);- let targetName = probeName;- if (collided) {- targetName = `${probeName}-imported`;- let n = 2;- while (nameTaken(targetName)) {- targetName = `${probeName}-imported-${n}`;- n += 1;- }- }-- // 4. Write SKILL.md + every bundle file via the sanctioned writers. The- // frontmatter is canonicalized to {name,description} (OK's skill model);- // upstream version lives in the lockfile, not the SKILL.md.- const skillBody = parseFrontmatterDoc(acquired.skillMd).body;- const wr = applySkillWrite({- skillsRoot: importRoot,- name: targetName,- body: skillBody,- frontmatter: { name: targetName, description: acquired.description },- });- if (!wr.ok) {- return {- ok: false,- status: 400,- urn: 'urn:ok:error:invalid-request',- title: 'Failed to write imported skill.',- detail: wr.error.code,- cause: new Error(wr.error.message),- };- }- const warnings = [...wr.warnings];- for (const f of acquired.files) {- const br = applySkillBundleFileWrite({- skillsRoot: importRoot,- name: targetName,- relPath: f.relPath,- content: f.content,- bytes: f.bytes,- limits: SKILL_IMPORT_WRITE_LIMITS,- });- if (!br.ok) {- const rollback = applySkillDelete({ skillsRoot: importRoot, name: targetName });- return {- ok: false,- status: rollback.ok ? 400 : 500,- urn: rollback.ok ? 'urn:ok:error:invalid-request' : 'urn:ok:error:internal-server-error',- title: 'Failed to write the complete imported skill bundle.',- detail: `${f.relPath}: ${br.error.code}`,- cause: new Error(- rollback.ok- ? br.error.message- : `${br.error.message}; rollback: ${rollback.error.message}`,- ),- };- }- }-- // 5. Attribute + shadow-commit (project scope; global is unversioned).- if (scope === 'project') {- // Attribute under the content-doc key (`<dir>/SKILL`) that `/api/history`- // filters on — same fix as the create path; the bare dir key fell through- // the OkActor match so an imported skill's first version never showed in- // (or was restorable from) its history.- attributeOkArtifactWrite(- actor,- `${importHomeRel}/${targetName}/SKILL`,- `skill-import: ${sourceLabel} -> ${importHomeRel}/${targetName}/SKILL.md`,- );- await commitOkArtifactWrite('skill-import');- }-- // 6. Record upstream in the lockfile.- const importedLocalHash = localSkillHash(importRoot, targetName);- const importedPlugin = inspectPluginSource(sourceLabel);- // baselineRef only for project scope — global is unversioned (no commit above).- const importedBaselineRef =- scope === 'project'- ? await shadowHeadSha(artifactWriterId(actor), `${importHomeRel}/${targetName}`)- : undefined;- // Re-read inside the serialized mutation: `lock` above was snapshotted before- // the fetch, and writing that snapshot back would erase any entry a- // concurrent import added while this one was cloning.- await mutateSkillsLock(lockPath, (current) =>- upsertLockEntry(current, targetName, {- source: sourceLabel,- ...(importedPlugin ? { pluginProvider: importedPlugin.provider } : {}),- ref,- contentHash: acquired.contentHash,- files: acquired.files.map((file) => file.relPath),- ...(importedLocalHash !== undefined ? { localHash: importedLocalHash } : {}),- ...(importedBaselineRef !== undefined ? { baselineRef: importedBaselineRef } : {}),- publisher,- importedAt: new Date().toISOString(),- ...(upstreamSkill !== undefined ? { skill: upstreamSkill } : {}),- }),- );-- // Fan the freshly-imported IN-PLACE skill out to the configured editors so- // it is live in Claude/Cursor/Codex on import (guarded copy/link- // primitives — never clobbers a differing dir). Best-effort: a projection- // failure must not fail the import. Skipped when the caller installs- // explicitly afterwards (`install: false`). Copies auto-pair into the- // placements ledger at the next re-sync, so forward refresh covers them.- if (params.skipProjection !== true) {- try {- const targets = resolveSkillTargets(importBase);- if (targets.length > 0) {- const canonicalAbs = resolve(importRoot, targetName);- // A re-import lands on a skill that may already occupy editor dirs as- // copies; only a genuinely fresh one takes the symlink default.- const projectionRoots = skillProjectionRoots(scope);- // A re-import can land on dests that already exist; those that are- // symlinks keep the symlink default, matching what is already there.- const occupied = targets.filter((editor) => {- const dest = skillHostDir(importBase, editor, targetName, projectionRoots);- return dest !== null && dest !== canonicalAbs && existsSync(dest);- });- const occupiedLinks = occupied.filter((editor) => {- const dest = skillHostDir(importBase, editor, targetName, projectionRoots);- try {- return dest !== null && lstatSync(dest).isSymbolicLink();- } catch {- return false;- }- });- projectInPlaceSkill({- canonicalAbs,- canonicalHash: parseSkillDir(canonicalAbs)?.contentHash ?? '',- canonicalRootRel: importHomeRel,- name: targetName,- cwd: importBase,- targets,- mode: effectiveInstallMode(scope, targetName, {- hosts: ['<source>', ...occupied],- linkedHosts: occupiedLinks,- }),- roots: skillProjectionRoots(scope),- });- }- } catch (err) {- getLogger('skill-import').warn(- { err, skill: targetName },- 'post-import editor fan-out failed (import kept)',- );- }- }-- signalChannel?.('files');- return {- ok: true,- body: {- name: targetName,- path: `${importHomeRel}/${targetName}/SKILL.md`,- created: wr.created,- alreadyImported: false,- ...(collided ? { collisionRenamedFrom: acquired.name } : {}),- provenance: { source: sourceLabel, ref, contentHash: acquired.contentHash, publisher },- warnings,- },- };+ return;+ }+ errorResponse(res, outcome.status, outcome.urn, outcome.title, {+ handler: 'skill-import',+ ...(outcome.detail !== undefined ? { detail: outcome.detail } : {}),+ ...(outcome.cause !== undefined ? { cause: outcome.cause } : {}),+ });}+ const skillImportService = createSkillImportService({+ contentDir,+ skillsHome,+ ...(projectDir !== undefined ? { projectDir } : {}),+ resolveSkillDirForRead,+ parseFrontmatterDoc,+ attributeOkArtifactWrite,+ commitOkArtifactWrite,+ shadowHeadSha,+ artifactWriterId,+ effectiveInstallMode,+ signalFiles: () => signalChannel?.('files'),+ });const handleSkillImport = withValidation(SkillImportRequestSchema,async (_req, res, body) => {@@ -19567,7 +18189,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}// Steps 2-6 (dedupe → collision → write → attribute → lockfile → project)// are the shared import spine, also used by the upload endpoint.- const outcome = await runSkillImport({+ const outcome = await skillImportService.runSkillImport({acquiredDir,scope,sourceLabel,@@ -19754,7 +18376,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {upstreamSkill = found.name;}try {- const outcome = await runSkillImport({+ const outcome = await skillImportService.runSkillImport({acquiredDir,scope: body.scope,sourceLabel: rawSource,@@ -20141,7 +18763,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {const pick = dirs[0];respondSkillImport(res,- await runSkillImport({+ await skillImportService.runSkillImport({acquiredDir: pick.dir,scope,sourceLabel: `upload:${zipName ?? pick.name}`,@@ -20204,7 +18826,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {? scanInPlaceSkills(contentDir): scanGlobalInPlaceSkills(skillsHome)).find((s) => s.name === body.name);- let skillDir = inPlaceEntry ? resolve(inPlaceScanBase, inPlaceEntry.dir) : storeSkillDir;+ const skillDir = inPlaceEntry ? resolve(inPlaceScanBase, inPlaceEntry.dir) : storeSkillDir;if (!existsSync(skillDir)) {errorResponse(res, 404, 'urn:ok:error:not-found', 'Skill not found.', {handler: 'skill-install',@@ -20243,9 +18865,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// stashes the bytes it discards to `~/.ok/edit-backups/forks/` first —// resolution is never silently lossy.if (body.fork !== undefined) {- // Warnings raised while resolving a fork — surfaced on the response so a- // half-completed rename is not reported as a clean success.- const forkWarnings: string[] = [];if (!inPlaceEntry) {errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Skill is not in-place.', {handler: 'skill-install',@@ -20253,153 +18872,53 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {});return;}- const rootMap =- body.scope === 'project' ? EDITOR_PROJECT_SKILL_ROOT : EDITOR_USER_SKILL_ROOT;- const forkRootRel =- body.fork.editor === 'agents'- ? '.agents/skills'- : (rootMap[body.fork.editor as EditorId] ?? null);- if (forkRootRel === null) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Unknown editor.', {- handler: 'skill-install',- detail: body.fork.editor,- });- return;- }- const forkDir = resolve(inPlaceScanBase, forkRootRel, body.name);- const canonicalAbs = resolve(inPlaceScanBase, inPlaceEntry.dir);- const forkParsed = parseSkillDir(forkDir);- if (forkParsed === null || forkDir === canonicalAbs) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'No fork at that editor.', {- handler: 'skill-install',- detail: 'FORK_ABSENT',- });- return;- }- if (forkParsed.contentHash === inPlaceEntry.contentHash) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- 'That copy matches the source — nothing to resolve.',- { handler: 'skill-install', detail: 'NOT_A_FORK' },- );- return;- }- const stamp = new Date().toISOString().replace(/[:.]/g, '-');- const stash = (dirAbs: string, label: string): void => {- try {- const dest = resolve(- skillsHome,- '.ok',- 'edit-backups',- 'forks',- `${body.name}-${label}-${stamp}`,- );- tracedMkdirSync(resolve(dest, '..'), { recursive: true });- tracedCpSync(dirAbs, dest, { recursive: true });- } catch (e) {- log.warn({ name: body.name, err: e }, '[skill-fork] stash failed');- }- };- if (body.fork.action === 'align') {- // The fork loses: stash it, remove it, re-project the canonical.- stash(forkDir, body.fork.editor);- tracedRmSync(forkDir, { recursive: true, force: true });- projectInPlaceSkill({- canonicalAbs,- canonicalHash: inPlaceEntry.contentHash,- canonicalRootRel: dirname(inPlaceEntry.dir),- name: body.name,- cwd: inPlaceScanBase,- targets: [body.fork.editor as SkillHostId].filter(isSkillInstallTarget),- roots: skillProjectionRoots(body.scope),- });- } else if (body.fork.action === 'make-source') {- // The fork wins: stash + remove the old canonical and its same-hash- // copies; the fork's dir is then the only real dir and the scan- // elects it; re-project to the hosts the skill had.- const oldHosts = inPlaceEntry.hosts.filter(isSkillInstallTarget);- stash(canonicalAbs, 'source');- removeInPlaceSkillCopies({- canonicalAbs,- canonicalHash: inPlaceEntry.contentHash,- name: body.name,- cwd: inPlaceScanBase,- targets: oldHosts,- roots: skillProjectionRoots(body.scope),- });- tracedRmSync(canonicalAbs, { recursive: true, force: true });- const rescanned = (- body.scope === 'project'- ? scanInPlaceSkills(contentDir)- : scanGlobalInPlaceSkills(skillsHome)- ).find((sk) => sk.name === body.name);- if (rescanned) {- projectInPlaceSkill({- canonicalAbs: resolve(inPlaceScanBase, rescanned.dir),- canonicalHash: rescanned.contentHash,- canonicalRootRel: dirname(rescanned.dir),- name: body.name,- cwd: inPlaceScanBase,- targets: oldHosts,- roots: skillProjectionRoots(body.scope),- });- }- } else {- // Keep both: the fork's dir becomes an independent skill under a- // NEW name (frontmatter rewritten in lock-step with the dir).- const toName = body.fork.toName as string;- if (!SKILL_NAME_REGEX.test(toName)) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid new name.', {- handler: 'skill-install',- detail: toName,- });- return;- }- const scanAll =- body.scope === 'project'- ? scanInPlaceSkills(contentDir)- : scanGlobalInPlaceSkills(skillsHome);- if (scanAll.some((sk) => sk.name === toName)) {- errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Name already taken.', {- handler: 'skill-install',- detail: toName,- });- return;- }- const dest = resolve(inPlaceScanBase, forkRootRel, toName);- tracedRenameSync(forkDir, dest);- // The folder move already happened. If the frontmatter patch below- // fails the skill is left half-renamed, which the caller has to- // hear about — a 200 with no warnings reads as a clean rename.- const skillMdPath = resolve(dest, 'SKILL.md');- try {- // Patch the YAML region, not the file. A multiline `^name:` regex- // matches the first such line ANYWHERE, so a body line reading- // `name: …` (a code sample, a config snippet) got rewritten- // instead of the frontmatter. Same primitives the duplicate- // handler uses.- const raw = readFileSync(skillMdPath, 'utf-8');- const { fenced, body: skillBody } = detectFmRegion(raw);- // presence-exempt: no CRDT write, no agent identity- const renamed = applyPatchToFm(fenced, { name: toName });- if (renamed.ok) {- tracedWriteFileSync(skillMdPath, `${renamed.nextFenced}${skillBody}`);- } else {- log.warn(- { name: toName, reason: renamed.error.kind },- '[skill-fork] frontmatter rename failed',+ const forkResolved = skillInstallOps.resolveFork({+ scope: body.scope,+ name: body.name,+ fork: body.fork,+ inPlaceEntry,+ });+ if (!forkResolved.ok) {+ switch (forkResolved.kind) {+ case 'unknown-editor':+ errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Unknown editor.', {+ handler: 'skill-install',+ detail: forkResolved.editor,+ });+ return;+ case 'fork-absent':+ errorResponse(res, 400, 'urn:ok:error:invalid-request', 'No fork at that editor.', {+ handler: 'skill-install',+ detail: 'FORK_ABSENT',+ });+ return;+ case 'not-a-fork':+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ 'That copy matches the source — nothing to resolve.',+ { handler: 'skill-install', detail: 'NOT_A_FORK' },);- forkWarnings.push(- `Renamed the folder to "${toName}", but its SKILL.md still declares the old name — edit the frontmatter to match.`,+ return;+ case 'invalid-new-name':+ errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Invalid new name.', {+ handler: 'skill-install',+ detail: forkResolved.toName,+ });+ return;+ case 'name-taken':+ errorResponse(res, 400, 'urn:ok:error:invalid-request', 'Name already taken.', {+ handler: 'skill-install',+ detail: forkResolved.toName,+ });+ return;+ default: {+ const _exhaustive: never = forkResolved;+ throw new Error(+ `Unhandled fork outcome: ${String((_exhaustive as { kind?: unknown }).kind)}`,);}- } catch (e) {- log.warn({ name: toName, err: e }, '[skill-fork] frontmatter rename failed');- forkWarnings.push(- `Renamed the folder to "${toName}", but its SKILL.md could not be updated — edit the frontmatter to match.`,- );}}signalChannel?.('files');@@ -20411,8 +18930,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {name: body.name,hosts: inPlaceEntry.hosts.filter(isSkillInstallTarget),scripts: false,- warnings: forkWarnings,- warningCodes: forkWarnings.length > 0 ? ['skill-fork-name-unpatched'] : [],+ warnings: forkResolved.warnings,+ warningCodes: forkResolved.warnings.length > 0 ? ['skill-fork-name-unpatched'] : [],},{ handler: 'skill-install' },);@@ -20443,97 +18962,36 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- const stripHome = (id: string) => id.replace(/\\/g, '/').replace(/^~\//, '');- const sourceId = inPlaceEntry.hosts[0];- if (- sourceId !== undefined &&- (body.remove ?? []).some((id) => stripHome(id) === sourceId)- ) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- `"${sourceId}" is the skill's SOURCE — its folder is the skill itself, so removing it would delete the skill. Move the source first (\`source\`) or use \`delete\`.`,- { handler: 'skill-install', detail: 'REMOVE_SOURCE' },- );- return;- }- const hostSet = new Set(- inPlaceEntry.hosts.filter((h): h is SkillHostIdArg => isSkillInstallTarget(h)),- );- for (const id of body.add ?? []) {- if (isSkillInstallTarget(id)) hostSet.add(id);- else rootAdds.push(stripHome(id));- }- // A removed host that is NOT a physical location but READS the skill- // through a folder alias (its skills folder symlinked into a pool- // root holding the skill) gets the unfollow remedy automatically:- // its folder unlinks from the pool EXCLUDING this skill — one verb- // means "this agent stops getting the skill" however it got it.- const aliasMap = scanHostRootAliases(inPlaceScanBase, body.scope);- const aliasUnfollows: string[] = [];- // The ADD-side mirror: an added host whose skills folder is an alias- // into a pool that does NOT hold this skill would otherwise be a- // silent no-op (projection never writes through an alias). One verb- // means "this agent gets the skill" however its folder is wired — so- // the alias unlinks into per-skill links first (keeping everything it- // followed), and the projection below then writes the real bundle.- const aliasMaterializes: string[] = [];- for (const id of body.add ?? []) {- if (- isSkillInstallTarget(id) &&- aliasMap[id] !== undefined &&- !existsSync(resolve(inPlaceScanBase, aliasMap[id], body.name, 'SKILL.md'))- ) {- const subRoot =- id === 'agents'- ? '.agents/skills'- : ((body.scope === 'project'- ? EDITOR_PROJECT_SKILL_ROOT- : EDITOR_USER_SKILL_ROOT)[id as EditorId] ?? null);- if (subRoot !== null) aliasMaterializes.push(subRoot);- }- }- for (const id of body.remove ?? []) {- if (isSkillInstallTarget(id)) {- if (- !hostSet.has(id) &&- aliasMap[id] !== undefined &&- existsSync(resolve(inPlaceScanBase, aliasMap[id], body.name, 'SKILL.md'))- ) {- const subRoot =- id === 'agents'- ? '.agents/skills'- : ((body.scope === 'project'- ? EDITOR_PROJECT_SKILL_ROOT- : EDITOR_USER_SKILL_ROOT)[id as EditorId] ?? null);- if (subRoot !== null) aliasUnfollows.push(subRoot);- }- hostSet.delete(id);- } else rootRemoves.push(stripHome(id));- }- for (const { subRoot, exclude } of [- ...aliasMaterializes.map((subRoot) => ({ subRoot, exclude: undefined })),- ...aliasUnfollows.map((subRoot) => ({ subRoot, exclude: [body.name] })),- ]) {- const r = unlinkEditorSkillFolder({- base: inPlaceScanBase,- folderRel: subRoot,- ...(exclude !== undefined ? { exclude } : {}),- });- if (r.ok) await recordFolderExpectation(inPlaceScanBase, subRoot, { expect: 'own' });- if (!r.ok) {+ const addRemove = await skillInstallOps.applyAddRemove({+ scope: body.scope,+ name: body.name,+ inPlaceEntry,+ ...(body.add !== undefined ? { add: body.add } : {}),+ ...(body.remove !== undefined ? { remove: body.remove } : {}),+ });+ if (!addRemove.ok) {+ if (addRemove.kind === 'remove-source') {+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ `"${addRemove.sourceId}" is the skill's SOURCE — its folder is the skill itself, so removing it would delete the skill. Move the source first (\`source\`) or use \`delete\`.`,+ { handler: 'skill-install', detail: 'REMOVE_SOURCE' },+ );+ } else {errorResponse(res,409,'urn:ok:error:invalid-request',- `Could not stop ${subRoot} following its pool (${r.reason}).`,- { handler: 'skill-install', detail: r.reason },+ `Could not stop ${addRemove.subRoot} following its pool (${addRemove.reason}).`,+ { handler: 'skill-install', detail: addRemove.reason },);- return;}+ return;}- targetsReq = [...hostSet];+ targetsReq = addRemove.targets.filter(isSkillInstallTarget);+ rootAdds.push(...addRemove.rootAdds);+ rootRemoves.push(...addRemove.rootRemoves);}// SOURCE promotion for a STORE-BACKED skill — a single-skill version of@@ -20543,44 +19001,38 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// entry drops (the in-place scan is truth from here on), and the// choice is sticky. Without this branch the click silently no-opped.if (setSourceReq && !inPlaceEntry) {- const newSource = setSourceReq as SkillHostId;- const moved = relocateInPlaceCanonical({- canonicalAbs: skillDir,- canonicalHash: parseSkillDir(skillDir)?.contentHash ?? '',+ const promoted = await skillInstallOps.promoteStoreBackedSource({+ scope: body.scope,name: body.name,- cwd: inPlaceScanBase,- newTarget: newSource,- roots: skillProjectionRoots(body.scope),+ base,+ skillDir,+ newSource: setSourceReq as SkillHostId,});- if (!moved.ok) {+ if (!promoted.ok) {errorResponse(res,409,'urn:ok:error:doc-already-exists','Cannot move the source there — a different skill occupies the target.',- { handler: 'skill-install', detail: moved.reason },+ {+ handler: 'skill-install',+ detail: promoted.kind === 'source-occupied' ? promoted.reason : promoted.target,+ },);return;}- await removeSkillInstall(base, body.name);- await recordSkillSourceHost(inPlaceScanBase, body.name, newSource);signalChannel?.('files');- const postPromote = (- body.scope === 'project'- ? scanInPlaceSkills(contentDir)- : scanGlobalInPlaceSkills(skillsHome)- ).find((s) => s.name === body.name);successResponse(res,200,SkillInstallSuccessSchema,{name: body.name,- hosts: postPromote ? [...postPromote.hosts] : [],+ hosts: promoted.hosts,scripts: validity.hasScripts,warnings: [],warningCodes: [],- sourceMovedTo: relative(inPlaceScanBase, moved.newAbs).split(sep).join('/'),+ sourceMovedTo: promoted.sourceMovedTo,},{ handler: 'skill-install' },);@@ -20607,80 +19059,36 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- const dirRel = body.place.dir- .replace(/\\/g, '/')- .replace(/^~\//, '')- .replace(/^\/+|\/+$/g, '');- const parentAbs = resolve(placeBase, dirRel);- const placementRel = `${dirRel}/${body.name}`;- const destAbs = resolveSkillPlacementPath(placeBase, placementRel);- const baseAbs = resolve(placeBase);- // `.ok/` internals are refused; `.ok/skills` is placeable at both- // scopes. See `isRefusedOkPlacementRoot` for why that one path is- // carved out.- const underOkInternals = isRefusedOkPlacementRoot(dirRel);- if (- dirRel === '' ||- destAbs === null ||- !destAbs.startsWith(baseAbs + sep) ||- underOkInternals- ) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- 'Placement path must be a project-relative directory outside .ok/.',- { handler: 'skill-install', detail: 'PLACE_PATH_INVALID' },- );- return;- }- // Asking for the location the skill ALREADY occupies is a satisfied- // request, not a bad path. Since imports land in-place at the- // `.agents/skills` hub, placing a freshly imported skill there names- // its own canonical dir — every caller (the hub toggle on a preview,- // `install --place`, MCP) hit an invalid-path error for a skill that- // was, in fact, exactly where it was asked to be. Report the location- // and change nothing; a real copy/symlink here would be self-referential.- if (destAbs === resolve(skillDir)) {- successResponse(- res,- 200,- SkillInstallSuccessSchema,- {- name: body.name,- hosts: inPlaceEntry ? [...inPlaceEntry.hosts] : [],- scripts: validity.hasScripts,- warnings: [],- warningCodes: [],- placedAt: placementRel,- },- { handler: 'skill-install' },- );- return;- }- if (existsSync(destAbs)) {- errorResponse(- res,- 409,- 'urn:ok:error:doc-already-exists',- 'Something already exists at that path — placement never overwrites.',- { handler: 'skill-install', detail: 'PLACE_DEST_EXISTS' },- );+ const placed = await skillPlacementOps.place({+ placeBase,+ name: body.name,+ rawDir: body.place.dir,+ skillDir,+ mode: body.place.mode,+ });+ if (!placed.ok) {+ if (placed.kind === 'invalid-path') {+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ 'Placement path must be a project-relative directory outside .ok/.',+ { handler: 'skill-install', detail: 'PLACE_PATH_INVALID' },+ );+ } else {+ errorResponse(+ res,+ 409,+ 'urn:ok:error:doc-already-exists',+ 'Something already exists at that path — placement never overwrites.',+ { handler: 'skill-install', detail: 'PLACE_DEST_EXISTS' },+ );+ }return;}- tracedMkdirSync(parentAbs, { recursive: true });- if (body.place.mode === 'link') {- tracedSymlinkSync(relative(parentAbs, resolve(skillDir)), destAbs, 'dir');- } else {- tracedCpSync(skillDir, destAbs, { recursive: true, dereference: true });+ if (!('alreadyAtSource' in placed)) {+ signalChannel?.('files');}- const placedAt = placementRel;- await recordSkillPlacement(placeBase, body.name, {- path: placedAt,- mode: body.place.mode,- ...(body.place.mode === 'copy' ? { hash: parseSkillDir(skillDir)?.contentHash } : {}),- });- signalChannel?.('files');successResponse(res,200,@@ -20691,7 +19099,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {scripts: validity.hasScripts,warnings: [],warningCodes: [],- placedAt,+ placedAt: placed.placedAt,},{ handler: 'skill-install' },);@@ -20714,58 +19122,58 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- const rel = body.unplace.path- .replace(/\\/g, '/')- .replace(/^~\//, '')- .replace(/^\/+|\/+$/g, '');- const recorded = readSkillPlacements(placeBase)[body.name]?.find((p) => p.path === rel);- if (!recorded) {- errorResponse(- res,- 404,- 'urn:ok:error:not-found',- 'No recorded placement at that path.',- { handler: 'skill-install', detail: rel },- );- return;- }- const absDir = resolveSkillPlacementPath(placeBase, rel);- if (absDir === null) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- 'Recorded placement path is no longer safe.',- { handler: 'skill-install', detail: 'PLACE_PATH_INVALID' },- );- return;- }- const canonicalHash = parseSkillDir(skillDir)?.contentHash ?? '';- switch (classifyInPlaceDest(absDir, resolve(skillDir), canonicalHash)) {- case 'different':- errorResponse(- res,- 409,- 'urn:ok:error:doc-already-exists',- 'That copy has been edited and no longer matches the skill — remove it manually if you mean it.',- { handler: 'skill-install', detail: rel },- );- return;- case 'canonical-dir':- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- "That is the skill's own folder (the source) — it can't be removed here.",- { handler: 'skill-install', detail: rel },- );- return;- case 'absent':- break; // already gone — just drop the record- default:- tracedRmSync(absDir, { recursive: true, force: true });+ const unplaced = await skillPlacementOps.unplace({+ placeBase,+ name: body.name,+ rawPath: body.unplace.path,+ skillDir,+ });+ if (!unplaced.ok) {+ switch (unplaced.kind) {+ case 'not-recorded':+ errorResponse(+ res,+ 404,+ 'urn:ok:error:not-found',+ 'No recorded placement at that path.',+ { handler: 'skill-install', detail: unplaced.path },+ );+ return;+ case 'unsafe-path':+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ 'Recorded placement path is no longer safe.',+ { handler: 'skill-install', detail: 'PLACE_PATH_INVALID' },+ );+ return;+ case 'forked':+ errorResponse(+ res,+ 409,+ 'urn:ok:error:doc-already-exists',+ 'That copy has been edited and no longer matches the skill — remove it manually if you mean it.',+ { handler: 'skill-install', detail: unplaced.path },+ );+ return;+ case 'canonical-dir':+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ "That is the skill's own folder (the source) — it can't be removed here.",+ { handler: 'skill-install', detail: unplaced.path },+ );+ return;+ default: {+ const _exhaustive: never = unplaced;+ throw new Error(+ `Unhandled unplace outcome: ${String((_exhaustive as { kind?: unknown }).kind)}`,+ );+ }+ }}- await removeSkillPlacement(placeBase, body.name, rel);signalChannel?.('files');successResponse(res,@@ -20803,91 +19211,60 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {}const { target, mode } = body.convert;const prefBase = body.scope === 'project' ? projectDir : skillsHome;- const roots = skillProjectionRoots(body.scope);- const isHost = isSkillInstallTarget(target);- const rootRel = isHost- ? target === 'agents'- ? '.agents/skills'- : roots[target as EditorId]- : target- .replace(/\\/g, '/')- .replace(/^~\//, '')- .replace(/^\/+|\/+$/g, '');- const absDir =- rootRel === null || rootRel === ''- ? null- : resolveSkillPlacementPath(prefBase ?? base, `${rootRel}/${body.name}`);- if (rootRel === null || rootRel === '' || absDir === null) {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- 'That location has no skills folder to convert.',- { handler: 'skill-install', detail: target },- );- return;- }- const canonicalAbs = resolve(skillDir);- const canonicalHash = inPlaceEntry.contentHash;- const cls = classifyInPlaceDest(absDir, canonicalAbs, canonicalHash);- if (cls === 'canonical-dir') {- errorResponse(- res,- 400,- 'urn:ok:error:invalid-request',- "That is the skill's own folder (the source) — move the source instead of converting it.",- { handler: 'skill-install', detail: target },- );- return;- }- if (cls === 'different') {- errorResponse(- res,- 409,- 'urn:ok:error:doc-already-exists',- 'That copy has been edited and no longer matches the skill — resolve the fork before converting it.',- { handler: 'skill-install', detail: target },- );- return;- }- if (cls === 'absent') {- errorResponse(res, 404, 'urn:ok:error:not-found', 'The skill is not installed there.', {- handler: 'skill-install',- detail: target,- });- return;- }- // Already in the requested form — nothing to write (a link-to-somewhere-- // else still gets re-materialized, same as the fan-out treats it).- const alreadyRight =- (mode === 'link' && cls === 'link-to-canonical') ||- (mode === 'copy' && cls === 'same-copy');- if (alreadyRight) {- // Disk is right but the ledger may not be, and a record that- // disagrees with disk is exactly what renders "changed outside".- // Skipping the write here used to skip the record too, so a stale- // record could never be reconciled from the UI — converting again- // just no-opped. Recording is idempotent when the two agree.- await recordSkillPlacement(prefBase ?? base, body.name, {- path: `${rootRel}/${body.name}`,- mode,- ...(mode === 'copy' ? { hash: canonicalHash } : {}),- });- }- if (!alreadyRight) {- const hostRoot = dirname(absDir);- tracedRmSync(absDir, { recursive: true, force: true });- tracedMkdirSync(hostRoot, { recursive: true });- if (mode === 'link') {- tracedSymlinkSync(relative(hostRoot, canonicalAbs), absDir, 'dir');- } else {- tracedCpSync(canonicalAbs, absDir, { recursive: true, dereference: true });+ const converted = await skillPlacementOps.convert({+ ledgerBase: prefBase ?? base,+ scope: body.scope,+ name: body.name,+ target,+ mode,+ skillDir,+ canonicalHash: inPlaceEntry.contentHash,+ });+ if (!converted.ok) {+ switch (converted.kind) {+ case 'invalid-location':+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ 'That location has no skills folder to convert.',+ { handler: 'skill-install', detail: target },+ );+ return;+ case 'canonical-dir':+ errorResponse(+ res,+ 400,+ 'urn:ok:error:invalid-request',+ "That is the skill's own folder (the source) — move the source instead of converting it.",+ { handler: 'skill-install', detail: target },+ );+ return;+ case 'forked':+ errorResponse(+ res,+ 409,+ 'urn:ok:error:doc-already-exists',+ 'That copy has been edited and no longer matches the skill — resolve the fork before converting it.',+ { handler: 'skill-install', detail: target },+ );+ return;+ case 'not-installed':+ errorResponse(+ res,+ 404,+ 'urn:ok:error:not-found',+ 'The skill is not installed there.',+ { handler: 'skill-install', detail: target },+ );+ return;+ default: {+ const _exhaustive: never = converted;+ throw new Error(+ `Unhandled convert outcome: ${String((_exhaustive as { kind?: unknown }).kind)}`,+ );+ }}- await recordSkillPlacement(prefBase ?? base, body.name, {- path: `${rootRel}/${body.name}`,- mode,- ...(mode === 'copy' ? { hash: canonicalHash } : {}),- });}signalChannel?.('files');successResponse(@@ -20991,359 +19368,82 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {// links everywhere else (the old source stays installed, as a link).// Sticky so the next scan doesn't re-elect by precedence.if (setSourceReq) {- const currentSource = inPlaceEntry.hosts[0] as string | undefined;- const newSource = setSourceReq;- // Editor ids + the hub map to host dirs; anything else is a CUSTOM- // skills-root path (its synthetic host id IS the root path).- const isHostTarget = isSkillInstallTarget(newSource);- let customDestAbs: string | undefined;- if (!isHostTarget) {- const rel = newSource- .replace(/\\/g, '/')- .replace(/^~\//, '')- .replace(/^\/+|\/+$/g, '');- const invalidOk = isRefusedOkPlacementRoot(rel);- const destAbs = resolve(base, rel, body.name);- if (rel === '' || invalidOk || !destAbs.startsWith(resolve(base) + sep)) {+ const promoted = await skillInstallOps.promoteInPlaceSource({+ scope: body.scope,+ name: body.name,+ base,+ ...(prefBase ? { prefBase } : {}),+ skillDir,+ inPlaceEntry,+ newSource: setSourceReq,+ });+ if (!promoted.ok) {+ if (promoted.kind === 'invalid-target') {errorResponse(res,400,'urn:ok:error:invalid-request','Source target must be an editor id, "agents", or a project-relative skills root.',- { handler: 'skill-install', detail: newSource },+ { handler: 'skill-install', detail: promoted.target },);- return;- }- customDestAbs = destAbs;- }- let sourceMovedTo: string | undefined;- if (currentSource !== undefined && newSource !== currentSource) {- const oldSourceRel = relative(inPlaceScanBase, resolve(skillDir))- .split(sep)- .join('/');- const moved = relocateInPlaceCanonical({- canonicalAbs: skillDir,- canonicalHash: inPlaceEntry.contentHash,- name: body.name,- cwd: base,- newTarget: (isHostTarget ? newSource : 'agents') as SkillHostId,- ...(customDestAbs !== undefined ? { destDirAbs: customDestAbs } : {}),- // The promote/downgrade SWAP: the old source path becomes a- // symlink to the new one (set-source is never a removal).- leaveLinkBehind: true,- roots: skillProjectionRoots(body.scope),- });- if (!moved.ok) {+ } else {errorResponse(res,409,'urn:ok:error:doc-already-exists','Cannot move the source there — a different skill occupies the target.',- { handler: 'skill-install', detail: moved.reason },+ { handler: 'skill-install', detail: promoted.reason },);- return;- }- sourceMovedTo = relative(inPlaceScanBase, moved.newAbs).split(sep).join('/');- // The swap left a symlink at the old source path (relocate's- // `leaveLinkBehind`). RECORD it as the expected form — without- // the receipt, an external tool rewriting that link as a copy- // would be invisible to drift detection (the .agents blind spot).- await recordSkillPlacement(prefBase ?? base, body.name, {- path: oldSourceRel,- mode: 'link',- });- // Re-point recorded placement SYMLINKS that referenced the old- // source (the relocation loop only covers host dirs).- for (const p of readSkillPlacements(prefBase ?? base)[body.name] ?? []) {- const abs = resolve(prefBase ?? base, p.path);- if (abs === moved.newAbs) continue;- try {- if (!lstatSync(abs).isSymbolicLink()) continue;- } catch {- continue;- }- let target: string | null = null;- try {- target = realpathSync(abs);- } catch {- target = null; // dangling (pointed at the moved-away dir)- }- if (target === null || target === resolve(skillDir)) {- tracedRmSync(abs, { recursive: true, force: true });- tracedSymlinkSync(relative(dirname(abs), moved.newAbs), abs, 'dir');- }}- // The promoted path is now the REAL dir — refresh its receipt to- // the promoted form (hash-less copy: no drift, resync skips it,- // and a custom root stays registered with the scan). A leftover- // 'link' expectation would misread the promotion as drift- // (OVERWRITTEN on a folder OK itself just wrote), inviting an- // accidental hands-off.- await recordSkillPlacement(prefBase ?? base, body.name, {- path: sourceMovedTo,- mode: 'copy',- });- await recordSkillSourceHost(prefBase ?? base, body.name, newSource);+ return;}signalChannel?.('files');- const postMove = (- body.scope === 'project'- ? scanInPlaceSkills(contentDir)- : scanGlobalInPlaceSkills(skillsHome)- ).find((sk) => sk.name === body.name);successResponse(res,200,SkillInstallSuccessSchema,{name: body.name,- hosts: postMove ? [...postMove.hosts] : [...inPlaceEntry.hosts],+ hosts: promoted.hosts,scripts: validity.hasScripts,warnings: [],warningCodes: [],- ...(sourceMovedTo !== undefined ? { sourceMovedTo } : {}),+ ...(promoted.sourceMovedTo !== undefined+ ? { sourceMovedTo: promoted.sourceMovedTo }+ : {}),},{ handler: 'skill-install' },);return;}- const prior = inPlaceEntry.hosts.filter((h): h is SkillHostId => isSkillInstallTarget(h));- // Unchecking THE SOURCE relocates it: the bundle moves to the- // highest-precedence remaining target (its copy/link there becomes- // the real dir), sibling links re-point, and set-exact proceeds- // against the new source. Refused when no other target remains —- // removing the only location is delete, not uninstall.- // `targets: []` (uninstall everywhere) is NOT a relocation request:- // lossless copies strip below and the source survives, matching the- // never-delete invariant.- // A CUSTOM-ROOT source (path-like host id, e.g. `.ok/skills`) is- // inexpressible in the editor-checkbox target vocabulary, so its- // absence from a set-exact list is NOT an uncheck — a mode flip- // (Copies/Symlinks re-apply) must never relocate it. Only an- // editor/hub source can be unchecked, and only that relocates.- // Relocation needs an EXPLICIT uncheck. With `targets` omitted the- // caller expressed no opinion about membership at all — the list came- // from `resolveSkillTargets`, which never includes the `agents` hub, so- // a defaults-driven call would read the hub's absence as an uncheck and- // move the real folder out from under it. Same reasoning as the two- // carve-outs above: `targets: []` and a custom-root source are excluded- // because neither is an uncheck either.- const sourceHost = inPlaceEntry.hosts[0] as SkillHostId | undefined;- let sourceMovedTo: string | undefined;- if (- targetsReq !== undefined &&- sourceHost !== undefined &&- !sourceHost.includes('/') &&- inPlaceTargets.length > 0 &&- !inPlaceTargets.includes(sourceHost)- ) {- const newTarget = inPlaceTargets[0] as SkillHostId;- const moved = relocateInPlaceCanonical({- canonicalAbs: skillDir,- canonicalHash: inPlaceEntry.contentHash,- name: body.name,- cwd: base,- newTarget,- roots: skillProjectionRoots(body.scope),- });- if (!moved.ok) {- errorResponse(- res,- 409,- 'urn:ok:error:doc-already-exists',- 'Cannot move the source there — a different skill occupies the target.',- { handler: 'skill-install', detail: moved.reason },- );- return;- }- skillDir = moved.newAbs;- sourceMovedTo = relative(inPlaceScanBase, moved.newAbs).split(sep).join('/');- // The path holding the source is now the REAL dir — a stale 'link'- // receipt there would misread the relocation as drift.- await recordSkillPlacement(prefBase ?? base, body.name, {- path: sourceMovedTo,- mode: 'copy',- });- // Sticky: without this the next scan re-elects the source by- // static precedence and the user's choice silently reverts.- await recordSkillSourceHost(prefBase ?? base, body.name, newTarget);- }- const effectiveRootRel = sourceMovedTo- ? sourceMovedTo.split('/').slice(0, -1).join('/')- : canonicalRootRel;-- removeInPlaceSkillCopies({- canonicalAbs: skillDir,- canonicalHash: inPlaceEntry.contentHash,- name: body.name,- cwd: base,- // Set-exact removal only when the caller actually named a set. With- // `targets` omitted this call is additive ("make sure it is in my- // configured editors"), so subtracting the locations that list- // happens not to mention would delete work nobody asked to remove.- targets:- targetsReq === undefined ? [] : prior.filter((h) => !inPlaceTargets.includes(h)),- roots: skillProjectionRoots(body.scope),- });- const fanned = projectInPlaceSkill({- canonicalAbs: skillDir,- canonicalHash: inPlaceEntry.contentHash,- canonicalRootRel: effectiveRootRel,+ const fanOut = await skillInstallOps.fanOutInPlace({+ scope: body.scope,name: body.name,- cwd: base,- targets: inPlaceTargets,- mode: installMode,- // An EXPLICIT copy choice converts existing links back to copies- // (lossless — the link's bytes ARE the canonical's); an implicit- // copy default never touches links.- convertLinks: linkModeReq === false,- roots: skillProjectionRoots(body.scope),+ base,+ ...(prefBase ? { prefBase } : {}),+ skillDir,+ inPlaceEntry,+ canonicalRootRel,+ inPlaceTargets,+ setExact: targetsReq !== undefined,+ installMode,+ ...(linkModeReq !== undefined ? { linkModeReq } : {}),+ rootAdds,+ rootRemoves,});- for (const editor of fanned.conflicted) {- warnings.push(- `A different skill named "${body.name}" already exists in the ${editor} skills folder — left untouched.`,+ if (!fanOut.ok) {+ errorResponse(+ res,+ 409,+ 'urn:ok:error:doc-already-exists',+ 'Cannot move the source there — a different skill occupies the target.',+ { handler: 'skill-install', detail: fanOut.reason },);- warningCodes.push('name-conflict');- }- // Custom placements the call is ADDING adopt the requested form; ones- // it does not name are left alone. A blanket flip of every recorded- // placement as a side effect of installing is exactly what the app- // retired — converting is its own verb, per location, and asks first.- if (linkModeReq !== undefined && prefBase && rootAdds.length > 0) {- const adding = new Set(rootAdds.map((r) => `${r}/${body.name}`));- for (const p of (readSkillPlacements(prefBase)[body.name] ?? []).filter((p) =>- adding.has(p.path),- )) {- const abs = resolve(prefBase, p.path);- if (abs === resolve(skillDir)) continue;- const cls = classifyInPlaceDest(abs, resolve(skillDir), inPlaceEntry.contentHash);- if (linkModeReq === true && cls === 'same-copy') {- tracedRmSync(abs, { recursive: true, force: true });- tracedSymlinkSync(relative(dirname(abs), resolve(skillDir)), abs, 'dir');- await recordSkillPlacement(prefBase, body.name, { path: p.path, mode: 'link' });- } else if (linkModeReq === false && cls === 'link-to-canonical') {- tracedRmSync(abs, { recursive: true, force: true });- tracedCpSync(resolve(skillDir), abs, { recursive: true, dereference: true });- await recordSkillPlacement(prefBase, body.name, {- path: p.path,- mode: 'copy',- hash: inPlaceEntry.contentHash,- });- }- }- }-- // Record what we just WROTE (machine-local): copies so the forward- // re-sync can lossless-refresh them, links as the EXPECTED form so- // drift detection can spot another tool rewriting the path.- for (const editor of fanned.hosts) {- const editorRoot =- editor === 'agents' ? '.agents/skills' : EDITOR_PROJECT_SKILL_ROOT[editor];- if (editorRoot === null || editorRoot === canonicalRootRel) continue;- const copyAbs = resolve(base, editorRoot, body.name);- let isLink = false;- try {- isLink = lstatSync(copyAbs).isSymbolicLink();- } catch {- continue;- }- if (isLink) {- if (installMode === 'link') {- await recordSkillPlacement(prefBase ?? base, body.name, {- path: `${editorRoot}/${body.name}`,- mode: 'link',- });- }- continue;- }- const copyHash = existsSync(join(copyAbs, 'SKILL.md'))- ? parseSkillDir(copyAbs)?.contentHash- : undefined;- if (copyHash !== undefined && copyHash === inPlaceEntry.contentHash) {- await recordSkillPlacement(prefBase ?? base, body.name, {- path: `${editorRoot}/${body.name}`,- mode: 'copy',- hash: copyHash,- });- }- }- // ── Additive custom-root placements (path-like `add`/`remove`- // members). Idempotent: an already-satisfied add is a no-op; a- // hand-edited fork is refused with a warning, never deleted.- if ((rootAdds.length > 0 || rootRemoves.length > 0) && prefBase) {- const canonAbs = resolve(skillDir);- for (const rootRel of rootAdds) {- const underOk = isRefusedOkPlacementRoot(rootRel);- const parentAbs = resolve(prefBase, rootRel);- // Symlink-aware containment, the same resolver `place` / `unplace`- // / `convert` use. A lexical `startsWith` prefix test walks- // straight through a checked-in symlink (`.team ->- // ~/Library/LaunchAgents`), and the mkdir + cp below would then- // write the bundle outside the project entirely.- const destAbs = resolveSkillPlacementPath(prefBase, `${rootRel}/${body.name}`);- if (rootRel === '' || underOk || destAbs === null || destAbs === canonAbs) {- warnings.push(`"${rootRel}" is not a placeable custom root — skipped.`);- warningCodes.push('place-path-invalid');- continue;- }- const cls = classifyInPlaceDest(destAbs, canonAbs, inPlaceEntry.contentHash);- if (cls === 'different') {- warnings.push(- `A different "${body.name}" already exists at ${rootRel} — left untouched.`,- );- warningCodes.push('name-conflict');- continue;- }- if (cls === 'absent') {- tracedMkdirSync(parentAbs, { recursive: true });- if (installMode === 'link') {- tracedSymlinkSync(relative(parentAbs, canonAbs), destAbs, 'dir');- } else {- tracedCpSync(canonAbs, destAbs, { recursive: true, dereference: true });- }- }- await recordSkillPlacement(prefBase, body.name, {- path: `${rootRel}/${body.name}`,- mode: installMode,- ...(installMode === 'copy' ? { hash: inPlaceEntry.contentHash } : {}),- });- }- for (const rootRel of rootRemoves) {- const rel = `${rootRel}/${body.name}`;- const recorded = readSkillPlacements(prefBase)[body.name]?.find(- (pl) => pl.path === rel,- );- const absDir = resolveSkillPlacementPath(prefBase, rel);- if (absDir === null) {- warnings.push(`"${rootRel}" is not a placeable custom root — skipped.`);- warningCodes.push('place-path-invalid');- continue;- }- const cls = classifyInPlaceDest(absDir, canonAbs, inPlaceEntry.contentHash);- if (cls === 'different') {- warnings.push(- `The copy at ${rel} has been hand-edited (a fork) — refused, never deleted. Remove it manually if you mean it.`,- );- warningCodes.push('place-fork-refused');- continue;- }- if (cls !== 'absent' && cls !== 'canonical-dir') {- tracedRmSync(absDir, { recursive: true, force: true });- }- if (recorded) await removeSkillPlacement(prefBase, body.name, rel);- }+ return;}-- // Report the FULL honest post-op host set by re-scanning — covers- // hub adds/removes, canonical-protection skips, and capability-covered- // hosts, so the client's before/after diff never invents changes.- const postEntry = (- body.scope === 'project'- ? scanInPlaceSkills(contentDir)- : scanGlobalInPlaceSkills(skillsHome)- ).find((s) => s.name === body.name);- const postHosts = postEntry ? [...postEntry.hosts] : fanned.hosts;+ warnings.push(...fanOut.warnings);+ warningCodes.push(...fanOut.warningCodes);signalChannel?.('files');successResponse(res,@@ -21351,11 +19451,13 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {SkillInstallSuccessSchema,{name: body.name,- hosts: postHosts,+ hosts: fanOut.hosts,scripts: validity.hasScripts,warnings,warningCodes,- ...(sourceMovedTo !== undefined ? { sourceMovedTo } : {}),+ ...(fanOut.sourceMovedTo !== undefined+ ? { sourceMovedTo: fanOut.sourceMovedTo }+ : {}),},{ handler: 'skill-install' },);@@ -22371,47 +20473,6 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {{ handler: 'skill-revert', method: 'POST' },);- function deriveFolderSearchDocuments(- pages: readonly WorkspaceSearchDocument[],- ): WorkspaceSearchDocument[] {- const folderModified = new Map<string, number>();- for (const page of pages) {- const segments = page.path.split('/').filter(Boolean);- segments.pop();- for (let i = 1; i <= segments.length; i++) {- const folderPath = segments.slice(0, i).join('/');- folderModified.set(- folderPath,- Math.max(folderModified.get(folderPath) ?? 0, page.modifiedTs),- );- }- }- return [...folderModified.entries()].map(([path, modifiedTs]) =>- createWorkspaceSearchDocument({ kind: 'folder', path, modifiedTs }),- );- }-- function buildSearchSnippet(content: string, query: string): string | undefined {- const normalizedQuery = query.trim().toLowerCase();- if (!normalizedQuery || !content) return undefined;- const normalizedContent = content.toLowerCase();- const index = normalizedContent.indexOf(normalizedQuery);- if (index < 0) return undefined;- const start = Math.max(0, index - 80);- const end = Math.min(content.length, index + normalizedQuery.length + 120);- const prefix = start > 0 ? '…' : '';- const suffix = end < content.length ? '…' : '';- // slice() cuts on UTF-16 code units, so a boundary landing mid-emoji leaves a- // lone surrogate. Replace any unpaired surrogate with U+FFFD so strict JSON-RPC- // clients (Rust / pydantic parsers) don't reject the response as invalid UTF-8.- // (String.toWellFormed() would do this but needs the es2024 lib in every consumer.)- const snippet = `${prefix}${content.slice(start, end).replace(/\s+/g, ' ').trim()}${suffix}`;- return snippet.replace(- /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,- '\uFFFD',- );- }-function parseSearchIntent(value: unknown): WorkspaceSearchIntent {if (value === 'autocomplete' || value === 'full_text' || value === 'omnibar') return value;return 'omnibar';@@ -22441,631 +20502,19 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {return value === 'omnibar' || value === 'mcp' || value === 'http' ? value : 'http';}- interface SemanticResolution {- /** Vector input for `searchWorkspaceCorpus`, or undefined for pure-lexical. */- input?: WorkspaceSemanticInput;- /** Non-content coverage status block to attach to the response. */- status?: SearchSemanticStatus;- /** Per-query embed latency (ms), or null when no query embed ran. */- queryEmbedMs: number | null;- /** Total embeddable pages (coverage denominator). */- pageTotal: number;- /** Whether the embedder is loaded + keyed + warm. */- capable: boolean;- }-- /**- * Resolve the per-query vector signal + coverage status for a search.- *- * Returns a pure-lexical resolution (no `input`, no `status`) — byte-identical- * to the pre-embeddings path — unless the feature flag is ON **and** the caller- * opted in (`semantic: true`). The omnibar and `semantic: false` never opt in,- * so they stay lexical and carry no status block. When opted-in, fires the lazy- * background corpus embed (no-op when incapable) and embeds only the query.- */- async function resolveSemantic(- query: string,- intent: WorkspaceSearchIntent,- semanticParam: boolean | undefined,- corpus: WorkspaceSearchCorpus,- ): Promise<SemanticResolution> {- // Predicate split: hidden / dot-path docs are searchable (admitted to the- // corpus) but NEVER embedded — no semantic egress for agent-tooling/dotfiles.- // The embeddable set is the corpus minus hidden paths, and it also drives the- // coverage denominator so a searchable dot-path page is never counted as- // "embeddable" (which would make coverage under-report forever).- const embeddableDocs = corpus.documents.filter((d) => !isHiddenDocName(d.path));- const pageTotal = embeddableDocs.reduce((n, d) => n + (d.kind === 'page' ? 1 : 0), 0);- // Flag OFF, or the caller did not opt in → no status block, lexical path.- if (!semanticSearch?.isEnabled() || semanticParam !== true) {- return { queryEmbedMs: null, pageTotal, capable: false };- }-- // Opted in + enabled: lazily (re-)embed the corpus in the background. Cheap- // for unchanged docs; no-op when no key. This is the only embed trigger —- // nothing embeds until an agent actually searches (no proactive egress).- void semanticSearch.embedCorpus(embeddableDocs);-- // Semantic fuses into the body blend only, and skips trivially short queries.- let input: WorkspaceSemanticInput | undefined;- let queryEmbedMs: number | null = null;- if (intent === 'full_text' && query.trim().length >= SEMANTIC_MIN_QUERY_LENGTH) {- const startedAt = performance.now();- const scores = await semanticSearch.queryScores(query, embeddableDocs);- queryEmbedMs = performance.now() - startedAt;- if (scores && scores.size > 0) {- // Carry the project-local similarity floor when set so a model whose- // cosine scale differs from the default can be retuned without a code- // change; undefined leaves core on its model-calibrated default.- const similarityFloor = getSemanticSimilarityFloor?.();- input = similarityFloor !== undefined ? { scores, similarityFloor } : { scores };- }- }-- const status = semanticSearch.getStatus();- return {- input,- status: {- capable: status.capable,- applied: false, // finalized post-ranking (did any result carry a vector)- coverage: { embedded: status.embeddedCount, total: pageTotal },- },- queryEmbedMs,- pageTotal,- capable: status.capable,- };- }-- /** Map a search result to the wire entry, carrying `vector` only when present. */- function toSearchResultEntry(- result: ReturnType<typeof searchWorkspaceCorpus>[number],- query: string,- ): {- kind: WorkspaceSearchScope;- path: string;- title: string;- score: number;- signals: WorkspaceSearchResult['signals'];- snippet?: string;- } {- return {- kind: result.document.kind,- path: result.document.path,- title: result.document.title,- score: result.score,- signals: result.signals,- snippet:- result.document.kind === 'page'- ? buildSearchSnippet(result.document.content, query)- : undefined,- };- }-- /**- * Shared core for `GET` + `POST /api/search`: build the corpus, resolve the- * (opt-in) vector signal, rank, and assemble the `SearchSuccess` body. One- * implementation so GET and POST cannot drift in ranking, snippets, or the- * semantic gate.- */- async function buildSearchResponse(params: {- query: string;- intent: WorkspaceSearchIntent;- ranking: WorkspaceSearchRanking | undefined;- scopes: WorkspaceSearchScope[] | undefined;- limit: number | undefined;- semanticParam: boolean | undefined;- source: SearchSource;- }): Promise<SearchSuccess> {- const startedAt = performance.now();- // Cold start: while the boot seed is still walking the content dir, do not- // block on it and do not serve a partial/empty index as if it were complete.- // Answer fast with `ready: false` so the caller (MCP `search`, palette, any- // consumer) retries instead of trusting an empty result. The seed populates- // the file index, so a retry after it resolves takes the normal path below.- if (isSearchCorpusWarming()) {- return {- query: params.query,- intent: params.intent,- results: [],- elapsedMs: Math.max(0, performance.now() - startedAt),- ready: false,- };- }- const { corpus, truncated } = await getWorkspaceSearchCorpus();- const semantic = await resolveSemantic(- params.query,- params.intent,- params.semanticParam,- corpus,- );- const results = searchWorkspaceCorpus(corpus, params.query, {- intent: params.intent,- ranking: params.ranking,- scopes: params.scopes,- limit: params.limit,- semantic: semantic.input,- });- const entries = results.map((r) => toSearchResultEntry(r, params.query));-- let semanticStatus: SearchSemanticStatus | undefined;- if (semantic.status) {- const vectorContributors = entries.reduce(- (n, e) => n + (e.signals.vector !== undefined ? 1 : 0),- 0,- );- const applied = vectorContributors > 0;- semanticStatus = { ...semantic.status, applied };- const outcome: SemanticQueryOutcome = !semantic.capable- ? 'incapable'- : applied- ? 'applied'- : semantic.status.coverage.embedded === 0- ? 'warming'- : 'no_match';- recordSemanticQuery({- outcome,- source: params.source,- capable: semantic.capable,- embedded: semantic.status.coverage.embedded,- total: semantic.pageTotal,- queryEmbedMs: semantic.queryEmbedMs,- vectorContributors,- });- }-- return {- query: params.query,- intent: params.intent,- results: entries,- elapsedMs: Math.max(0, performance.now() - startedAt),- ready: true,- ...(semanticStatus ? { semantic: semanticStatus } : {}),- ...(truncated ? { truncated: true } : {}),- };- }-- /**- * Project skills (`<root>/.ok/skills/<name>/SKILL.md`) as cheap stat records —- * readdir + stat only, no content read — so the per-search corpus fingerprint- * can detect skill changes without paying a content read on every request.- * Skills are tree-excluded from `getFileIndex()`, so search enumerates them- * from disk. The corpus doc builder reuses this list and reads each matched- * file's content.- */- function enumerateProjectSkillStats(): Array<{- name: string;- absolutePath: string;- /** The skill's LIVE content-doc path (`<real dir>/SKILL`) — what search- * hits open. In-place skills carry their editor-dir path; only a legacy- * store resident still carries the `.ok/skills` shape. */- docPath: string;- mtimeMs: number;- size: number;- }> {- const out: Array<{- name: string;- absolutePath: string;- docPath: string;- mtimeMs: number;- size: number;- }> = [];- const seen = new Set<string>();- // In-place skills FIRST (they win a name collision with a store resident,- // mirroring the list/read rules). Enumerating only the store root here was- // a store-retirement fossil: in-place skills vanished from search entirely,- // and the ones indexed opened phantom `.ok/skills` tabs.- for (const skill of scanInPlaceSkills(contentDir)) {- const skillMd = resolve(contentDir, skill.dir, 'SKILL.md');- try {- const st = statSync(skillMd);- out.push({- name: skill.name,- absolutePath: skillMd,- docPath: `${skill.dir}/SKILL`,- mtimeMs: st.mtimeMs,- size: st.size,- });- seen.add(skill.name);- } catch {- // Vanished between scan and stat — skip.- }- }- const root = resolveSkillsRoot('project');- if (!existsSync(root)) return out;- let entries: Dirent[];- try {- entries = readdirSync(root, { withFileTypes: true });- } catch {- return out;- }- for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {- if (!entry.isDirectory() || !SKILL_NAME_REGEX.test(entry.name)) continue;- if (seen.has(entry.name)) continue;- const skillMd = resolve(root, entry.name, 'SKILL.md');- try {- const st = statSync(skillMd);- out.push({- name: entry.name,- absolutePath: skillMd,- docPath: `${LEGACY_SKILL_STORE_ROOT}/${entry.name}/SKILL`,- mtimeMs: st.mtimeMs,- size: st.size,- });- } catch {- // Missing/unreadable SKILL.md — skip (a draft dir with no manifest).- }- }- return out;- }-- /**- * Project skills as search documents (keyword + semantic — `embedCorpus`- * embeds every corpus doc). Indexed under their managed-artifact doc path so a- * hit opens the skill tab via the shared nav resolution. Title is the skill's- * frontmatter name; content is its description + body, so a skill is findable- * by what it does, not just its slug.- */- function buildSkillSearchDocuments(): WorkspaceSearchDocument[] {- const docs: WorkspaceSearchDocument[] = [];- for (const skill of enumerateProjectSkillStats()) {- let title = skill.name;- let content = '';- try {- const { frontmatter, body } = parseFrontmatterDoc(- readFileSync(skill.absolutePath, 'utf-8'),- );- if (typeof frontmatter.name === 'string' && frontmatter.name) title = frontmatter.name;- const desc = typeof frontmatter.description === 'string' ? frontmatter.description : '';- content = `${desc}\n\n${body}`.trim();- } catch {- // Malformed/unreadable — index by name only so it is still findable.- }- docs.push(- createWorkspaceSearchDocument({- kind: 'page',- // Index under the skill's LIVE content-doc path (real dir), never a- // minted shape — a minted `__skill__/project/<name>` OR `.ok/skills`- // path both made a search hit open a blank phantom tab.- path: skill.docPath,- title,- content,- modifiedTs: skill.mtimeMs,- }),- );- }- return docs;- }-- // Per-entry change-detection key: the fields whose change should re-read a- // page (modified / size / canonical path / inode / aliases), NUL-separated so- // a path containing spaces can't merge fields and collide. `workspaceSearchFingerprint`'s- // fallback prefixes this with the docName; the page-doc cache keys on it- // directly (its Map is already docName-keyed). One definition keeps the two in- // lockstep — drift would silently break cache invalidation (stale reuse or- // needless re-reads).- function entrySearchKey(entry: FileIndexEntry): string {- // NUL between fields AND between aliases: a path/alias containing a comma- // (rare but valid on macOS/Linux) must not collide with a different alias set.- return `${entry.modified}\0${entry.size}\0${entry.canonicalPath}\0${entry.inode}\0${entry.aliases.join('\0')}`;- }-- // Per-page parsed-document cache. Building the corpus re-reads every markdown- // file from disk, but a rebuild is triggered by ANY file-index change (one- // edit, a rename, a new sibling), so without this every keystroke-after-an-edit- // would re-read and re-parse the whole workspace. Reuse a page's search- // document across rebuilds when its own entry is unchanged — re-reading only- // the delta. Invariant direction: a change that busts THIS page's `entrySearchKey`- // also bumps the generation counter that invalidates the corpus — but NOT the- // converse: a rebuild triggered by a sibling change reuses this page's cached- // doc when its own entry is unchanged (the whole point). Only successful reads- // are cached, so a transient read failure self-heals on the next rebuild rather- // than pinning empty content. Pruned to the live index each build, so it stays- // bounded by the workspace size. The name-only `file` tier and derived folder- // docs are metadata-only (no disk read), so they are rebuilt each time.- const pageDocCache = new Map<string, { key: string; doc: WorkspaceSearchDocument }>();-- async function buildWorkspaceSearchDocumentsFromIndex(): Promise<{- documents: WorkspaceSearchDocument[];- truncated: boolean;- }> {- const pages: WorkspaceSearchDocument[] = [];- const files: WorkspaceSearchDocument[] = [];- // Type-annotated, like the two siblings above, so the getAllFilesIndex- // caller-coverage meta-test attributes the call below to this (allowlisted)- // function rather than latching onto a bare local declaration.- const seenPages: Set<string> = new Set();- for (const [docName, entry] of getAllFilesIndex()) {- // System + config synthetic docs never enter search. Hidden / dot-prefixed- // paths (`.changeset/`, `.github/`, `.cursor/`) DO — they are searchable by- // name/path (rank-deprioritized in core) so "search what the tree shows"- // holds. They stay out of the embedding/egress path, which keeps the- // `isHiddenDocName` filter where the corpus is handed to the embedder.- if (isSystemDoc(docName) || isConfigDoc(docName)) continue;- // Project-skill SKILL docs ARE in the index (skills-as-content), and this- // loop iterates the all-files view — but `buildSkillSearchDocuments()`- // already indexes each skill with skill-aware title/content under the same- // path. Skip ANY project SKILL-doc shape (in-place editor dirs AND the- // legacy store) so one isn't added twice (a duplicate corpus id throws and- // 500s the whole search). Bundle reference docs stay — they index as- // ordinary pages.- if (parseProjectSkillBundleDoc(docName)?.kind === 'skill') continue;- if (docName.startsWith('.ok/skills/')) continue;- if (entry.kind === 'file') {- // Name-only tier: a non-markdown file is searchable by name / path /- // folder, but its body is NEVER read (content stays markdown-only).- // `pathToDocName` keeps the extension for non-markdown, so `data.csv`- // is findable by both `data` and `data.csv`; the basename is the title.- files.push(- createWorkspaceSearchDocument({- kind: 'file',- path: docName,- modifiedTs: Date.parse(entry.modified),- // Symlink alias paths fold into searchable pathSegments (inode-dedup- // already gives one entry per file via the canonical-keyed index).- aliases: entry.aliases,- }),- );- continue;- }- // Markdown page: reuse the cached parse when its entry is unchanged (same- // fingerprint components), else re-read and re-cache.- seenPages.add(docName);- const entryKey = entrySearchKey(entry);- const cached = pageDocCache.get(docName);- if (cached && cached.key === entryKey) {- pages.push(cached.doc);- continue;- }- let content = '';- let title = docName;- let readFailed = false;- try {- content = await readFile(entry.canonicalPath, 'utf-8');- } catch (err) {- // A transient read (external editor mid-save, EBUSY, NFS blip, a- // watcher-vs-disk race) must NOT be cached — the entry fingerprint does- // not change just because the read failed, so a cached empty-content doc- // would persist and silently hide the page from body search until its- // mtime/size/inode shifts. Skip the cache write so the next rebuild- // retries, preserving the pre-cache self-healing behavior.- readFailed = true;- log.warn({ docName, err }, `[search] Failed to read ${docName}`);- }- if (!readFailed) {- try {- title = extractPageTitle(content, docName);- } catch (err) {- // Title extraction is pure string work, so a throw here is a- // deterministic parse fault, not transient I/O. Fall back to the- // docName as title but still cache (the read succeeded) — caching it- // avoids re-parsing the same failing content on every rebuild, the- // opposite of the read-failure path's deliberate retry.- log.warn({ docName, err }, `[search] Failed to extract title for ${docName}`);- }- }- const doc = createWorkspaceSearchDocument({- kind: 'page',- path: docName,- title,- content,- modifiedTs: Date.parse(entry.modified),- aliases: entry.aliases,- });- if (!readFailed) pageDocCache.set(docName, { key: entryKey, doc });- pages.push(doc);- }- // Prune cache entries for pages no longer in the index (deleted / renamed)- // so the cache tracks the live workspace rather than growing unbounded.- // Unconditional: a failed read adds to `seenPages` but not to the cache, so- // a `size`-comparison guard could read equal and skip a genuinely-needed- // prune. The loop is O(cache) — same order as the build it follows.- for (const docName of pageDocCache.keys()) {- if (!seenPages.has(docName)) pageDocCache.delete(docName);- }- // Cap the name-only file tier (markdown pages are never dropped). Over the- // ceiling, drop DEEPEST paths first (level-order): the shallowest entries are- // the most navigationally useful, and dropping the deep tail mirrors the- // show-all truncation BFS. The dogfood repo (~16k) is far under the- // 50k default; this is a pathological-repo backstop.- const maxFiles = getSearchMaxEntries();- let admittedFiles = files;- let truncated = false;- if (files.length > maxFiles) {- truncated = true;- admittedFiles = [...files]- .sort((a, b) => {- const depthA = a.path.split('/').length;- const depthB = b.path.split('/').length;- return depthA - depthB || a.path.localeCompare(b.path);- })- .slice(0, maxFiles);- // Surface the cap-fire to operators: a structured warn log + a meter- // counter. Without these the cap is silent — operators see "search- // missing some files" with no signal pointing at `OK_SEARCH_MAX_ENTRIES`.- // One emission per corpus rebuild (the cache then absorbs subsequent- // queries until the fingerprint changes).- getLogger('search').warn(- {- dropped: files.length - admittedFiles.length,- retained: admittedFiles.length,- limit: maxFiles,- },- '[search] corpus name-only file tier truncated at OK_SEARCH_MAX_ENTRIES',- );- searchCorpusTruncatedCounter().add(1);- }- // Folders are synthesized from ALL admitted paths (markdown pages + name-only- // file entries), so a folder containing only non-markdown files is still a- // search result and a partial-path query (e.g. `server/src`) resolves even- // when the folder holds no markdown.- const documents = [- ...pages,- ...buildSkillSearchDocuments(),- ...admittedFiles,- ...deriveFolderSearchDocuments([...pages, ...admittedFiles]),- ];- return { documents, truncated };- }-- // Stat-only skill fingerprint (name + mtime + size per project skill). A- // named helper, not a local `const`, so the getAllFilesIndex caller-coverage- // meta-test attributes the call in `workspaceSearchFingerprint` to that- // allowlisted function rather than to an intermediate binding.- function skillStatFingerprint(): string {- return enumerateProjectSkillStats()- .map((s) => `${s.name} ${s.mtimeMs} ${s.size}`)- .join('');- }-- function workspaceSearchFingerprint(): string {- // Skills are tree-excluded from the file index, so neither the generation- // counter nor getAllFilesIndex reflects a skill add/edit/remove. Fold the- // stat-only skill fingerprint into BOTH paths so the corpus rebuilds on a- // skill change (no content read on the per-search fingerprint path).- // Fast path: the watcher's monotonic generation counter bumps on every- // file-index mutation (the same counter that memoizes the markdown-only- // view), so a generation match proves the corpus is still valid in O(1).- if (getFileIndexGeneration) {- return `gen:${getFileIndexGeneration()}|skills${skillStatFingerprint()}`;- }- // Fallback for harnesses that wire only the index accessors. Admission- // predicate MUST match `buildWorkspaceSearchDocumentsFromIndex` so a- // change to a now-searchable dot-path busts the corpus cache.- return `${[...getAllFilesIndex()]- .filter(([docName]) => !isSystemDoc(docName) && !isConfigDoc(docName))- .sort(([a], [b]) => a.localeCompare(b))- .map(- // Shares `entrySearchKey` with the page-doc cache so the two never drift.- ([docName, entry]) => `${docName}\0${entrySearchKey(entry)}`,- )- .join('')}|skills${skillStatFingerprint()}`;- }-- // Cold-start search readiness. While the boot index seed is still walking the- // content dir, `/api/search` must not block on it nor return a false-empty- // result: an agent (MCP `search`) or any consumer hitting search right after- // `ok start` would otherwise get zero hits that read as complete. We answer- // fast with `ready: false` instead and let the caller retry. The command- // palette gates its own fetch on the page-list cold-load signal, so this- // primarily protects non-UI consumers and is defense-in-depth for the UI.- //- // `bootIndexReady` mirrors the same boot gate `handleDocumentList` awaits: an- // absent gate (test harnesses) is ready immediately, and a rejected gate still- // flips ready (logged, like the sibling document-list gate) so a degraded boot- // serves whatever index exists rather than warming forever.- let bootIndexReady = ready === undefined;- ready?.then(- () => {- bootIndexReady = true;- },- (err: unknown) => {- bootIndexReady = true;- log.warn(- { err, handler: 'search' },- '[api] ready gate rejected — search serves the partial index',- );- },- );-- // Warming = the boot seed has not finished. Once it has, search awaits the- // corpus build and returns results as before (the lazy first build is fast and- // prewarmed; a slow first build on a very large workspace is the documented- // residual). Scoping warming to the seed window keeps steady-state behavior —- // and every consumer that does not pass a boot gate — unchanged.- function isSearchCorpusWarming(): boolean {- return !bootIndexReady;- }-- async function getWorkspaceSearchCorpus(): Promise<{- corpus: WorkspaceSearchCorpus;- truncated: boolean;- }> {- const cacheKey = `${contentDir} ${projectDir ?? ''}`;- const fingerprint = workspaceSearchFingerprint();- const workspaceSearchCache = workspaceSearchCaches.get(cacheKey);- if (workspaceSearchCache?.fingerprint === fingerprint && workspaceSearchCache.corpus) {- return {- corpus: workspaceSearchCache.corpus,- truncated: workspaceSearchCache.truncated ?? false,- };- }- if (workspaceSearchCache?.fingerprint === fingerprint && workspaceSearchCache.pending) {- return workspaceSearchCache.pending;- }-- // Stale-but-live corpus (or the build about to produce one): the base for- // an incremental index patch, so one write re-indexes one document instead- // of re-tokenizing the whole workspace on the event loop.- const priorCorpus = workspaceSearchCache?.corpus;- const priorPending = workspaceSearchCache?.pending;- const pending = (async () => {- // Serialize behind any in-flight build: an incremental diff is only valid- // against the corpus it was computed from, and the in-flight build owns- // the shared index right now. Chaining keeps updates linear (each build- // bases on its predecessor's output) without coalescing away freshness —- // the document snapshot below is read AFTER this fingerprint was seen.- const base = priorPending- ? await priorPending.then(- (result) => result.corpus,- () => undefined,- )- : priorCorpus;- const { documents, truncated } = await buildWorkspaceSearchDocumentsFromIndex();- if (!base) {- searchCorpusUpdateCounter().add(1, { mode: 'cold' });- return { corpus: createWorkspaceSearchCorpus(documents), truncated };- }- const update = updateWorkspaceSearchCorpus(base, documents);- if (update.rebuilt) {- searchCorpusUpdateCounter().add(1, { mode: 'rebuild', reason: update.rebuildReason });- // `mutation-failed` means the patched index diverged from the document- // set (or a mutation threw) — recovered by the rebuild, but worth an- // operator-visible signal; the elective reasons are routine.- const logLevel = update.rebuildReason === 'mutation-failed' ? 'warn' : 'debug';- getLogger('search')[logLevel](- { reason: update.rebuildReason, documents: documents.length },- '[search] corpus update fell back to a full index rebuild',- );- } else {- searchCorpusUpdateCounter().add(1, { mode: 'incremental' });- getLogger('search').debug(- { inserted: update.inserted, updated: update.updated, removed: update.removed },- '[search] corpus updated incrementally',- );- }- return { corpus: update.corpus, truncated };- })();- workspaceSearchCaches.set(cacheKey, { fingerprint, pending });- try {- const result = await pending;- if (workspaceSearchCaches.get(cacheKey)?.pending === pending) {- workspaceSearchCaches.set(cacheKey, {- fingerprint,- corpus: result.corpus,- truncated: result.truncated,- });- }- return result;- } catch (err) {- if (workspaceSearchCaches.get(cacheKey)?.pending === pending) {- workspaceSearchCaches.delete(cacheKey);- }- throw err;- }- }-- function prewarmWorkspaceSearchCache(): void {- if (process.env.NODE_ENV === 'test') return;- for (const delayMs of [0, 1000, 3000]) {- setTimeout(() => {- void getWorkspaceSearchCorpus().catch((err) => {- log.warn({ err }, '[search] Failed to prewarm workspace search cache');- });- }, delayMs);- }- }-- prewarmWorkspaceSearchCache();+ const searchService = createSearchService({+ contentDir,+ projectDir,+ getAllFilesIndex,+ getFileIndexGeneration,+ getSearchMaxEntries,+ semanticSearch,+ getSemanticSimilarityFloor,+ ready,+ getProjectSkillsRoot: () => resolveSkillsRoot('project'),+ parseFrontmatterDoc,+ });+ searchService.prewarm();const handleSearchGet = withValidation(EmptyRequestSchema,@@ -23091,7 +20540,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- const body = await buildSearchResponse({+ const body = await searchService.buildSearchResponse({query,intent,ranking,@@ -23129,7 +20578,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {);return;}- const responseBody = await buildSearchResponse({+ const responseBody = await searchService.buildSearchResponse({query,intent,ranking,diff --git a/packages/server/src/api-folder-config.test.ts b/packages/server/src/api-folder-config.test.tsnew file mode 100644index 000000000..9ed04c89b--- /dev/null+++ b/packages/server/src/api-folder-config.test.ts@@ -0,0 +1,236 @@+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { readFile } from 'node:fs/promises';+import type { IncomingMessage, ServerResponse } from 'node:http';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { Readable } from 'node:stream';+import { afterEach, beforeEach, describe, expect, test } from 'vitest';+import { createApiExtension } from './api-extension.test-helper.ts';++/**+ * Characterization for GET/PUT /api/folder-config ahead of the files/folders+ * extraction — neither method had any server-unit coverage. Pins the wire+ * contract: GET's self-only `frontmatter_local` (no ancestor cascade, null+ * when absent, {} when the YAML is a non-object, null again when malformed),+ * PUT's merge-patch outcomes (written/deleted/noop) landing in+ * `<folder>/.ok/frontmatter.yml`, the project-root-relative path gate, and+ * the single-file-mode refusal.+ */++interface CapturedResponse {+ status: number;+ headers: Record<string, string>;+ body: string;+}++function makeReq(url: string, method: string, body?: unknown): IncomingMessage {+ const raw = body === undefined ? '' : JSON.stringify(body);+ const readable = Readable.from(Buffer.from(raw)) as unknown as IncomingMessage;+ readable.method = method;+ readable.url = url;+ readable.headers = { host: 'localhost' };+ return readable;+}++function makeRes(): { res: ServerResponse; captured: CapturedResponse } {+ const captured: CapturedResponse = { status: 0, headers: {}, body: '' };+ const res = {+ writeHead(status: number, headers?: Record<string, string>) {+ captured.status = status;+ if (headers) Object.assign(captured.headers, headers);+ },+ end(body?: string) {+ captured.body = body ?? '';+ },+ } as unknown as ServerResponse;+ return { res, captured };+}++describe('GET/PUT /api/folder-config', () => {+ let tmpDir: string;+ let contentDir: string;++ function buildExt(extra?: { ephemeral?: boolean }) {+ return createApiExtension({+ hocuspocus: {} as Parameters<typeof createApiExtension>[0]['hocuspocus'],+ sessionManager: {} as Parameters<typeof createApiExtension>[0]['sessionManager'],+ contentDir,+ serverInstanceId: 'test-server',+ getFileIndex: () => new Map(),+ ...extra,+ }) as {+ onRequest: (ctx: { request: IncomingMessage; response: ServerResponse }) => Promise<void>;+ };+ }++ async function dispatch(+ ext: ReturnType<typeof buildExt>,+ url: string,+ method: string,+ body?: unknown,+ ): Promise<CapturedResponse> {+ const { res, captured } = makeRes();+ await ext.onRequest({ request: makeReq(url, method, body), response: res });+ return captured;+ }++ beforeEach(() => {+ tmpDir = mkdtempSync(join(tmpdir(), 'ok-folder-config-'));+ contentDir = join(tmpDir, 'content');+ mkdirSync(join(contentDir, 'docs', '.ok'), { recursive: true });+ mkdirSync(join(contentDir, 'plain'), { recursive: true });+ mkdirSync(join(contentDir, 'weird', '.ok'), { recursive: true });+ mkdirSync(join(contentDir, 'broken', '.ok'), { recursive: true });+ writeFileSync(+ join(contentDir, 'docs', '.ok', 'frontmatter.yml'),+ 'status: active\nowner: omar\n',+ );+ writeFileSync(join(contentDir, 'weird', '.ok', 'frontmatter.yml'), '- just\n- a list\n');+ writeFileSync(join(contentDir, 'broken', '.ok', 'frontmatter.yml'), 'key: [unclosed\n');+ });++ afterEach(() => {+ rmSync(tmpDir, { recursive: true, force: true });+ });++ test('GET returns the folder meta plus self-only frontmatter_local', async () => {+ const ext = buildExt();+ const captured = await dispatch(ext, '/api/folder-config?path=docs', 'GET');+ expect(captured.status).toBe(200);+ const body = JSON.parse(captured.body) as {+ folder: Record<string, unknown>;+ frontmatter_local: Record<string, unknown> | null;+ };+ expect(typeof body.folder).toBe('object');+ expect(body.frontmatter_local).toEqual({ status: 'active', owner: 'omar' });+ });++ test('GET reports null frontmatter_local when the sidecar is absent', async () => {+ const ext = buildExt();+ const captured = await dispatch(ext, '/api/folder-config?path=plain', 'GET');+ expect(captured.status).toBe(200);+ expect(JSON.parse(captured.body).frontmatter_local).toBeNull();+ });++ test('GET does not cascade an ancestor sidecar onto a sidecar-less child', async () => {+ // Pins the SELF-ONLY contract against a fixture where a cascade would be+ // observable: `docs/.ok/frontmatter.yml` exists, `docs/sub` has no+ // sidecar of its own. An accidental reuse of the template-resolution+ // ancestor walk would surface the parent's map here instead of null.+ mkdirSync(join(contentDir, 'docs', 'sub'), { recursive: true });+ const ext = buildExt();+ const captured = await dispatch(+ ext,+ `/api/folder-config?path=${encodeURIComponent('docs/sub')}`,+ 'GET',+ );+ expect(captured.status).toBe(200);+ expect(JSON.parse(captured.body).frontmatter_local).toBeNull();+ });++ test('GET flattens a non-object YAML document to {} and malformed YAML to null', async () => {+ const ext = buildExt();+ const list = await dispatch(ext, '/api/folder-config?path=weird', 'GET');+ expect(JSON.parse(list.body).frontmatter_local).toEqual({});++ const broken = await dispatch(ext, '/api/folder-config?path=broken', 'GET');+ expect(broken.status).toBe(200);+ expect(JSON.parse(broken.body).frontmatter_local).toBeNull();+ });++ test('GET rejects traversal and absolute paths as not project-root-relative', async () => {+ const ext = buildExt();+ const traversal = await dispatch(+ ext,+ `/api/folder-config?path=${encodeURIComponent('docs/../../outside')}`,+ 'GET',+ );+ expect(traversal.status).toBe(400);+ expect(JSON.parse(traversal.body).type).toBe('urn:ok:error:invalid-request');++ const absolute = await dispatch(+ ext,+ `/api/folder-config?path=${encodeURIComponent('/etc')}`,+ 'GET',+ );+ expect(absolute.status).toBe(400);+ });++ test('PUT writes the sidecar and reports written, then noop on an identical patch', async () => {+ const ext = buildExt();+ const first = await dispatch(ext, '/api/folder-config', 'PUT', {+ path: 'plain',+ frontmatter: { status: 'draft' },+ });+ expect(first.status).toBe(200);+ expect(JSON.parse(first.body).applied).toEqual([+ { path: 'plain/.ok/frontmatter.yml', action: 'written' },+ ]);+ const sidecar = join(contentDir, 'plain', '.ok', 'frontmatter.yml');+ expect(existsSync(sidecar)).toBe(true);+ expect(await readFile(sidecar, 'utf-8')).toContain('status: draft');++ // No diff detection: an identical re-patch (and even a delete of a+ // nonexistent key on an existing sidecar) rewrites and reports+ // 'written' again. 'noop' fires only when there is nothing to do at+ // all: a null-only patch against a folder with no sidecar.+ const second = await dispatch(ext, '/api/folder-config', 'PUT', {+ path: 'plain',+ frontmatter: { status: 'draft' },+ });+ expect(JSON.parse(second.body).applied).toEqual([+ { path: 'plain/.ok/frontmatter.yml', action: 'written' },+ ]);++ mkdirSync(join(contentDir, 'untouched'), { recursive: true });+ const noop = await dispatch(ext, '/api/folder-config', 'PUT', {+ path: 'untouched',+ frontmatter: { 'never-existed': null },+ });+ expect(JSON.parse(noop.body).applied).toEqual([+ { path: 'untouched/.ok/frontmatter.yml', action: 'noop' },+ ]);+ expect(existsSync(join(contentDir, 'untouched', '.ok'))).toBe(false);+ });++ test('PUT with null values deletes keys; clearing the last key deletes the sidecar', async () => {+ const ext = buildExt();+ const cleared = await dispatch(ext, '/api/folder-config', 'PUT', {+ path: 'docs',+ frontmatter: { status: null, owner: null },+ });+ expect(cleared.status).toBe(200);+ expect(JSON.parse(cleared.body).applied).toEqual([+ { path: 'docs/.ok/frontmatter.yml', action: 'deleted' },+ ]);+ expect(existsSync(join(contentDir, 'docs', '.ok', 'frontmatter.yml'))).toBe(false);+ });++ test('PUT without a frontmatter field applies nothing', async () => {+ const ext = buildExt();+ const captured = await dispatch(ext, '/api/folder-config', 'PUT', { path: 'plain' });+ expect(captured.status).toBe(200);+ expect(JSON.parse(captured.body).applied).toEqual([]);+ });++ test('PUT refuses in single-file mode with the dedicated URN', async () => {+ const ext = buildExt({ ephemeral: true });+ const captured = await dispatch(ext, '/api/folder-config', 'PUT', {+ path: 'plain',+ frontmatter: { a: 1 },+ });+ expect(captured.status).toBe(403);+ expect(JSON.parse(captured.body).type).toBe('urn:ok:error:single-file-mode');+ });++ test('PUT rejects a non-string summary before writing anything', async () => {+ const ext = buildExt();+ const captured = await dispatch(ext, '/api/folder-config', 'PUT', {+ path: 'plain',+ frontmatter: { a: 1 },+ summary: 42,+ });+ expect(captured.status).toBe(400);+ expect(existsSync(join(contentDir, 'plain', '.ok', 'frontmatter.yml'))).toBe(false);+ });+});diff --git a/packages/server/src/asset-serve-middleware.ts b/packages/server/src/asset-serve-middleware.tsindex a6c7dc973..dc86e4913 100644--- a/packages/server/src/asset-serve-middleware.ts+++ b/packages/server/src/asset-serve-middleware.ts@@ -56,92 +56,8 @@import type { IncomingMessage, ServerResponse } from 'node:http';import { extname } from 'node:path';-import { SANDBOXED_HTML_CSP, SANDBOXED_HTML_EXTENSIONS } from '@inkeep/open-knowledge-core';-import { mimes } from 'mrmime';--/**- * Close 3 gaps in mrmime's default mime table that break browser inline- * rendering for common user-drop formats. Without these, sirv serves the- * bytes with an empty `Content-Type` header — combined with our- * `Content-Disposition: inline` policy, Chromium renders the binary- * bytes as garbled text rather than dispatching to its built-in video /- * audio viewer.- *- * The fix is documented idiomatic usage per mrmime's README: "Exposes- * the `mimes` dictionary for easy additions or overrides." Three- * extensions need coverage:- *- * - `.m4v` → `video/mp4`. Apple's MP4 variant is structurally MP4;- * `video/mp4` is standards-recommended (WordPress Trac #24993,- * Mozilla bug 875573). mrmime deliberately filters `x-` types, so- * the historical `video/x-m4v` is not in its default table.- * - `.mkv` → `video/x-matroska`. De-facto type (no IANA registration- * exists); Chromium recognizes it. Only non-`x-` alternative would- * be `application/octet-stream` which blocks inline rendering.- * - `.flac` → `audio/flac`. IANA-registered (RFC 9639);- * `audio/x-flac` is the deprecated legacy alias.- *- * Security posture: setting extension-derived Content-Type on- * video/audio with `X-Content-Type-Options: nosniff` is NOT a stored-- * XSS vector. Browsers refuse to treat `video/*` / `audio/*` as- * scriptable regardless of file contents under nosniff (MDN- * X-Content-Type-Options, Beyond XSS ch5). The SVG polyglot class- * (`image/svg+xml`) is the real risk and is separately covered by- * `EXECUTABLE_BLOCKLIST_EXTENSIONS` barring `.svg` from the- * `openAssetSafely` click path.- *- * Module-load mutation runs once per Node process. Multiple dev-server- * invocations in the same process (Vite restart) re-assign idempotently.- *- * If a future inline-renderable extension lands without a mrmime entry,- * the narrow-integration test for `.m4v` will flag it (currently pinned- * to `video/mp4`). Extend this map in lockstep.- */-Object.assign(mimes, {- m4v: 'video/mp4',- mkv: 'video/x-matroska',- flac: 'audio/flac',- // TOML has an IANA-registered media type (`application/toml`) but- // `mrmime` doesn't ship it by default — the table is the- // narrow `mime-db` subset, not the full registry. Without this- // entry, sirv serves `.toml` with an empty `Content-Type` and our- // `/api/asset` handler 415s (the `assetContentTypeForPath` lookup- // returns null). The `TextViewer`'s own fetch path- // (`/api/asset-text`) forces `text/plain` and is therefore- // unaffected by this patch — what relies on it is the fallback- // pane's "Open file" link + any direct deeplink to a `.toml`- // asset URL. JSON is already covered by mrmime's defaults.- toml: 'application/toml',- // `.lock` has no IANA registration and no mrmime default. Same- // mrmime-gap pattern as `.toml` above; without this, the- // `INLINE_RENDERABLE_EXTENSIONS` widening for `lock` would 415 on- // direct `/api/asset?path=foo.lock` GETs. `text/plain` matches- // what the `TextViewer` path (`/api/asset-text`) already forces,- // so the sidebar-click and deeplink surfaces agree on the wire- // shape. Lockfile contents vary across ecosystems (some JSON-- // shaped, some custom DSLs) but `text/plain` is the right floor.- lock: 'text/plain',- // Attachment-only types newly admitted to ASSET_EXTENSIONS that `mrmime`'s- // default table omits. Without an entry, `assetContentTypeForPath` returns- // null and `handleAsset` 415s them (and sirv streams them with an empty- // Content-Type). All download-only — none are inline-renderable.- '7z': 'application/x-7z-compressed',- tar: 'application/x-tar',- rar: 'application/vnd.rar',- xls: 'application/vnd.ms-excel',- ppt: 'application/vnd.ms-powerpoint',- odt: 'application/vnd.oasis.opendocument.text',- ods: 'application/vnd.oasis.opendocument.spreadsheet',- odp: 'application/vnd.oasis.opendocument.presentation',- pages: 'application/vnd.apple.pages',- numbers: 'application/vnd.apple.numbers',- key: 'application/vnd.apple.keynote',- mobi: 'application/x-mobipocket-ebook',-});--export function assetContentTypeForPath(path: string): string | null {- return mimes[extname(path).slice(1).toLowerCase()] ?? null;-}+import { SANDBOXED_HTML_EXTENSIONS } from '@inkeep/open-knowledge-core';+import { classifyAssetDisposition } from './services/asset-classification.ts';/*** Minimal contract the middleware depends on. The real@@ -225,30 +141,16 @@ export function createAssetServeMiddleware(if (!rel || contentFilter.isPathIgnored(rel) || (!isDocExt && !assetExtensions.has(ext)))return next();res.setHeader('X-Content-Type-Options', 'nosniff');- // `html`/`htm` are admitted to ASSET_EXTENSIONS (so author-created HTML- // resolves + serves) but deliberately kept OUT of INLINE_RENDERABLE_EXTENSIONS.- // They render inline ONLY inside the sandbox CSP below — never as a plain- // same-origin document.- const isSandboxedHtml = SANDBOXED_HTML_EXTENSIONS.has(ext);+ // Shared with `/api/asset`: one extension → disposition/CSP policy for+ // both serve surfaces (rationale lives with the classifier).+ const classified = classifyAssetDisposition(ext, inlineExtensions);if (!isDocExt) {- if (inlineExtensions.has(ext) || isSandboxedHtml) {- res.setHeader('Content-Disposition', 'inline');- } else {- res.setHeader('Content-Disposition', 'attachment');- }+ res.setHeader('Content-Disposition', classified.disposition);+ }+ if (classified.csp !== null) {+ res.setHeader('Content-Security-Policy', classified.csp);}- // SVG served inline executes embedded `<script>` on a top-level navigation- // — `nosniff` doesn't help (`image/svg+xml` is CORB-excluded). Match the CSP- // sandbox `handleAsset` (api-extension.ts) applies so `GET /<rel>.svg` can't- // run scripts; the editor's `<img src>` render path is unaffected (embeds- // don't execute SVG).- if (ext === 'svg') {- res.setHeader(- 'Content-Security-Policy',- "sandbox; default-src 'none'; style-src 'unsafe-inline'",- );- } else if (isSandboxedHtml) {- res.setHeader('Content-Security-Policy', SANDBOXED_HTML_CSP);+ if (classified.sandboxedHtml) {// Match `/api/asset`'s no-store posture so an edited/removed sandboxed// document doesn't linger in the browser cache.res.setHeader('Cache-Control', 'no-store');diff --git a/packages/server/src/boot.test.ts b/packages/server/src/boot.test.tsindex 0c58043da..bd45a5e35 100644--- a/packages/server/src/boot.test.ts+++ b/packages/server/src/boot.test.ts@@ -49,7 +49,10 @@ beforeEach(async () => {});afterEach(async () => {- await rm(tmpDir, { recursive: true, force: true });+ // A just-destroyed server can still flush telemetry/log files into the+ // tmpdir for a beat, and rm's readdir-then-unlink walk races it into+ // ENOTEMPTY under parallel-suite load. Retries absorb the tail writes.+ await rm(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });});describe('bootServer — MissingOkConfigError pre-listen check', () => {diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex 9239a622d..872ca67e4 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -65,6 +65,7 @@ import { getLogger, loggerFactory, type PinoLogger } from './logger.ts';import { createMcpHttpHandler } from './mcp-http.ts';import { mountMcpAndApi, type ReadinessState } from './mcp-mount.ts';import { MissingOkConfigError } from './missing-ok-config-error.ts';+import { createProjectRuntime, type ProjectRuntime } from './project-runtime.ts';import { RemoteConfigError, resolveRemoteAccess } from './remote-access.ts';import { createServer, type ServerInstance, type ServerOptions } from './server-factory.ts';import { installServerMemoryGauge, installServerRuntimeGauges } from './server-memory-telemetry.ts';@@ -389,6 +390,12 @@ export interface BootedServer {didAutoInit: boolean;/** Full ServerInstance from createServer — exposed for advanced consumers (e.g., desktop utility's drain sequencing). */serverInstance: ServerInstance;+ /**+ * The project's stateful resources grouped behind the runtime boundary.+ * Capability services compose against this, not against `serverInstance`+ * internals; see `project-runtime.ts` for what belongs in versus out.+ */+ runtime: ProjectRuntime;/** ACP thread host, or null in ephemeral single-file mode. */acpThreadManager: AcpThreadManager | null;}@@ -1251,6 +1258,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {degraded,didAutoInit,serverInstance,+ runtime: createProjectRuntime(serverInstance, { contentDir: opts.contentDir, projectDir }),acpThreadManager,};}diff --git a/packages/server/src/content/managed-doc-enum.ts b/packages/server/src/content/managed-doc-enum.tsnew file mode 100644index 000000000..42ed4a244--- /dev/null+++ b/packages/server/src/content/managed-doc-enum.ts@@ -0,0 +1,75 @@+import { readdirSync, statSync } from 'node:fs';+import { extname, relative, resolve, sep } from 'node:path';+import { OK_DIR } from '@inkeep/open-knowledge-core';+import { isSupportedDocFile, registerDocExtension, stripDocExtension } from '../doc-extensions.ts';+import { errnoCode } from '../http/handler-utils.ts';++/**+ * Disk-truth enumeration of managed documents under a folder. Every+ * destructive or copying file operation (delete, duplicate, rename)+ * enumerates descendants from DISK, deliberately not from the lagging file+ * index — the disk-enum regression tests guard exactly that property. Shared+ * by the FileOps service and the managed-rename spine; do not fork it.+ */++/** True when any path segment is OK state (`.ok/`) or git internals. */+export function isReservedProjectStatePath(path: string): boolean {+ return path.split('/').some((segment) => {+ const normalized = segment.toLowerCase();+ return normalized === OK_DIR || normalized === '.git';+ });+}++export interface ManagedDocEnumDeps {+ contentDir: string;+ contentFilter?:+ | { isDirExcluded(relPath: string): boolean; isExcluded(relPath: string): boolean }+ | undefined;+ /** Maps a doc file's contentDir-relative path to its docName (extension policy lives with the caller). */+ docNameForPath: (relPath: string) => string;+}++export function listManagedDocNamesUnderFolder(+ sourcePathRoot: string,+ deps: ManagedDocEnumDeps,+): string[] {+ const { contentDir, contentFilter, docNameForPath } = deps;+ const docNames: string[] = [];+ // A file at the folder path (e.g. `kind: 'folder'` on a doc) must NOT reach+ // `readdirSync` — that throws ENOTDIR and 500s. Return empty for that and a+ // TOCTOU vanish (ENOENT) so the caller's type-mismatch / not-found check+ // emits the correct 4xx. Any other stat error (EACCES, EIO, ELOOP) means+ // the folder exists but is unreadable: returning empty there would move the+ // directory and skip link rewriting — the exact bug this fix addresses — so+ // rethrow and let it surface as a 500.+ try {+ if (!statSync(sourcePathRoot).isDirectory()) return docNames;+ } catch (err) {+ const code = errnoCode(err);+ if (code === 'ENOENT' || code === 'ENOTDIR') return docNames;+ throw err;+ }++ function walk(dir: string): void {+ for (const entry of readdirSync(dir, { withFileTypes: true })) {+ const fullPath = resolve(dir, entry.name);+ const relPath = relative(contentDir, fullPath).split(sep).join('/');+ if (isReservedProjectStatePath(relPath)) continue;+ if (entry.isDirectory()) {+ if (contentFilter?.isDirExcluded(relPath)) continue;+ walk(fullPath);+ continue;+ }+ if (!entry.isFile() || !isSupportedDocFile(relPath) || contentFilter?.isExcluded(relPath)) {+ continue;+ }+ const docName = docNameForPath(relPath);+ registerDocExtension(stripDocExtension(relPath), extname(relPath));+ docNames.push(docName);+ }+ }++ walk(sourcePathRoot);+ docNames.sort((a, b) => a.localeCompare(b));+ return docNames;+}diff --git a/packages/server/src/filename-sanitize.ts b/packages/server/src/filename-sanitize.tsnew file mode 100644index 000000000..a289fe4a4--- /dev/null+++ b/packages/server/src/filename-sanitize.ts@@ -0,0 +1,65 @@+/**+ * Portable filename sanitization for attacker-influenced names (multipart+ * upload filenames, imported SKILL.md frontmatter names). Pure string work,+ * shared across capability services so none depends on a sibling for it.+ */++// unicode-preserving. Permits any Unicode letter, number, or combining+// mark, plus pictographic emoji and the punctuation whitelist (., -, _, space).+// Everything else (including `/`, `\`, null bytes, control chars, CRLF) is+// either stripped or replaced so path-escape guards downstream keep their+// invariants. CJK, Arabic, Cyrillic, and emoji survive — macOS/Finder+// ergonomics without sacrificing filesystem safety.+const SAFE_FILENAME_CHARS = /[^\p{L}\p{N}\p{M}\p{Extended_Pictographic}.\-_ ]/gu;+// Stripping C0 + DEL is the whole point — the rule fires on intentional use.+// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional — sanitize must strip control bytes.+const STRIP_ON_SIGHT = /[/\\\x00-\x1f\x7f]/g;++export function sanitizeFilename(name: string): string {+ // Strip path separators and null/control bytes BEFORE any other pass so+ // they cannot reappear inside a replacement and dodge later checks.+ let stripped = name.replace(STRIP_ON_SIGHT, '');+ stripped = stripped.replace(SAFE_FILENAME_CHARS, '_');++ // Collapse underscore and dot runs so "../etc/passwd" → "etcpasswd" and+ // "foo__bar" → "foo_bar".+ stripped = stripped.replace(/_+/g, '_').replace(/\.{2,}/g, '.');++ // No hidden files — trim leading dots and leading underscores.+ stripped = stripped.replace(/^[._]+/, '');+ // Filesystem portability — strip trailing dots (Windows trims them too).+ stripped = stripped.replace(/\.+$/, '');++ if (stripped === '') return 'upload';++ // Most filesystems cap basenames at 255 bytes (ext4, APFS, exFAT). Without a+ // ceiling, a multipart `Content-Disposition` filename approaching busboy's+ // header size can sail through Unicode-letter sanitization and surface as+ // `ENAMETOOLONG` from `linkSync`, which classifies as a generic+ // `storage-error` → 500. Truncate the stem (preserving the extension) to+ // stay within the portable basename ceiling.+ const MAX_BYTES = 255;+ const encoder = new TextEncoder();+ if (encoder.encode(stripped).length > MAX_BYTES) {+ const dotIdx = stripped.lastIndexOf('.');+ const ext = dotIdx >= 0 ? stripped.slice(dotIdx) : '';+ let stem = dotIdx >= 0 ? stripped.slice(0, dotIdx) : stripped;+ // `slice(0, -1)` removes one UTF-16 code unit. A trailing emoji is a+ // surrogate pair, so the loop transiently produces a lone-surrogate+ // string that `TextEncoder` re-encodes as U+FFFD (3 bytes) — harmless+ // since the emoji is fully consumed before the loop exits and the+ // returned string is always valid UTF-8.+ while (encoder.encode(stem + ext).length > MAX_BYTES && stem.length > 0) {+ stem = stem.slice(0, -1);+ }+ stripped = (stem || 'upload') + ext;+ // The loop drains the stem; it cannot shrink the extension itself.+ // An adversarial 250+ byte extension (e.g. `'x.' + 'a'.repeat(300)`)+ // would drain the stem to empty and still leave `'upload' + ext`+ // above the ceiling. Final-pass guard: fall back to extensionless+ // `'upload'` when even the floor exceeds MAX_BYTES.+ if (encoder.encode(stripped).length > MAX_BYTES) stripped = 'upload';+ }++ return stripped;+}diff --git a/packages/server/src/fs-safety.ts b/packages/server/src/fs-safety.tsnew file mode 100644index 000000000..cb5dca66d--- /dev/null+++ b/packages/server/src/fs-safety.ts@@ -0,0 +1,87 @@+import { realpathSync } from 'node:fs';+import { dirname, sep } from 'node:path';+import { SymlinkEscapeError } from './apply-managed-rename.ts';+import { isWithinContentDir } from './content-path.ts';+import { errnoCode } from './http/handler-utils.ts';++/**+ * Filesystem-safety predicates shared by every write surface that creates or+ * moves entries under the content directory (uploads, file ops, renames).+ */++/**+ * Ensures `fullPath` does not escape `resolvedContentDir` via symlinks (matches persistence+ * symlink-escape checks). Walks up with dirname when the leaf is missing so destinations like+ * `link/new.md` are rejected if `link` resolves outside the content dir.+ *+ * Uses `realpathSync(resolvedContentDir)` as the boundary anchor so platform normalization+ * (e.g. macOS `/var` → `/private/var`) matches `realpathSync` of paths under it.+ */+export function assertNoSymlinkEscape(fullPath: string, resolvedContentDir: string): void {+ let contentRoot: string;+ try {+ contentRoot = realpathSync(resolvedContentDir);+ } catch (err) {+ const code = errnoCode(err);+ // ENOENT means the content dir hasn't been created yet — no symlink+ // escape is possible against a non-existent directory, but we have+ // no safe baseline for the check either. Throw the same+ // `symlink-escape:` error class so the caller's catch routes through+ // the existing error path. Other errno classes (EPERM, EIO, ENOMEM)+ // must NOT be swallowed silently — they'd leave the security gate+ // disabled with no log line, no telemetry, no error response. Throw+ // and let the top-level handler emit a typed RFC 9457 problem.+ if (code === 'ENOENT') {+ throw new SymlinkEscapeError('content directory does not exist');+ }+ throw err;+ }++ let cur = fullPath;+ for (;;) {+ try {+ const canonical = realpathSync(cur);+ if (!isWithinContentDir(canonical, contentRoot)) {+ throw new SymlinkEscapeError('path resolves outside content directory');+ }+ return;+ } catch (err) {+ const code = errnoCode(err);+ if (code === 'ELOOP') {+ throw new SymlinkEscapeError('symlink cycle in path');+ }+ if (code !== 'ENOENT') throw err;+ const parent = dirname(cur);+ if (parent === cur) throw err;+ // Bound the ENOENT ancestor walk against EITHER spelling of the root.+ // The walk's parents keep the caller's spelling, but a platform alias+ // (macOS /var → /private/var) can put the caller's root and the+ // realpath'd `contentRoot` in different spellings — matching only one+ // false-positives for the other. Widening is safe: this guard only+ // bounds how far the walk ascends; containment is enforced by the+ // realpath check above.+ const withinRaw =+ parent === resolvedContentDir || parent.startsWith(`${resolvedContentDir}${sep}`);+ const withinCanonical = parent === contentRoot || parent.startsWith(`${contentRoot}${sep}`);+ if (!withinRaw && !withinCanonical) {+ throw err;+ }+ cur = parent;+ }+ }+}++export function isAlreadyExistsError(err: unknown): boolean {+ const code = errnoCode(err);+ // Node's cpSync distinguishes the destination-occupied cases by the type+ // mismatch: a same-type collision surfaces as ERR_FS_CP_EEXIST, but copying a+ // directory onto an existing file (or a file onto an existing directory)+ // surfaces as ERR_FS_CP_DIR_TO_NON_DIR / ERR_FS_CP_NON_DIR_TO_DIR. All three+ // mean the destination path is already taken and must map to 409.+ return (+ code === 'EEXIST' ||+ code === 'ERR_FS_CP_EEXIST' ||+ code === 'ERR_FS_CP_DIR_TO_NON_DIR' ||+ code === 'ERR_FS_CP_NON_DIR_TO_DIR'+ );+}diff --git a/packages/server/src/project-runtime.test.ts b/packages/server/src/project-runtime.test.tsnew file mode 100644index 000000000..ba0ed4b85--- /dev/null+++ b/packages/server/src/project-runtime.test.ts@@ -0,0 +1,65 @@+import { mkdtemp, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { resolve } from 'node:path';+import { afterAll, beforeAll, describe, expect, test } from 'vitest';+import type { BootedServer } from './boot.ts';+import { bootCompositionRig } from './composition-rig.test-helper.ts';++/**+ * The runtime boundary's contract is that it is a pure grouping view:+ * every member IS the ServerInstance member (identity-preserved), and the+ * late-assigned members stay live through the view rather than being+ * snapshotted at construction. If either property breaks, adopting the+ * runtime in a capability service silently changes behavior.+ */++let tmpRoot: string;+let booted: BootedServer;++beforeAll(async () => {+ tmpRoot = await mkdtemp(resolve(tmpdir(), 'ok-runtime-'));+ booted = await bootCompositionRig(tmpRoot);+ await booted.ready;+}, 60_000);++afterAll(async () => {+ await booted?.destroy();+ await rm(tmpRoot, { recursive: true, force: true });+});++describe('ProjectRuntime view over ServerInstance', () => {+ test('grouped members are identity-preserved, not copies', () => {+ const { runtime, serverInstance } = booted;+ expect(runtime.crdt.hocuspocus).toBe(serverInstance.hocuspocus);+ expect(runtime.crdt.durabilityState).toBe(serverInstance.durabilityState);+ expect(runtime.crdt.cc1Broadcaster).toBe(serverInstance.cc1Broadcaster);+ expect(runtime.crdt.serverInstanceId).toBe(serverInstance.serverInstanceId);+ expect(runtime.content.contentFilter).toBe(serverInstance.contentFilter);+ expect(runtime.indexes.basenameIndex).toBe(serverInstance.basenameIndex);+ expect(runtime.indexes.resolveEmbed).toBe(serverInstance.resolveEmbed);+ expect(runtime.agents.sessionManager).toBe(serverInstance.sessionManager);+ expect(runtime.agents.agentFocusBroadcaster).toBe(serverInstance.agentFocusBroadcaster);+ expect(runtime.agents.agentPresenceBroadcaster).toBe(serverInstance.agentPresenceBroadcaster);+ expect(runtime.agents.acpRegistry).toBe(serverInstance.acpRegistry);+ expect(runtime.agents.acpPermissions).toBe(serverInstance.acpPermissions);+ expect(runtime.background.maintenanceCoordinator).toBe(serverInstance.maintenanceCoordinator);+ expect(runtime.config.getLinkPreviewsEnabled).toBe(serverInstance.getLinkPreviewsEnabled);+ expect(runtime.ready).toBe(serverInstance.ready);+ expect(runtime.destroy).toBe(serverInstance.destroy);+ });++ test('late-assigned members read live through the view', () => {+ const { runtime, serverInstance } = booted;+ // Both are getter-backed: whatever the instance reports now, the view+ // reports the same object at the same moment — no construction-time+ // snapshot (syncEngine is assigned after async init; degraded fills+ // during init).+ expect(runtime.git.syncEngine).toBe(serverInstance.syncEngine);+ expect(runtime.degraded).toBe(serverInstance.degraded);+ });++ test('path anchors come from the composer', () => {+ expect(booted.runtime.content.contentDir).toBe(booted.contentDir);+ expect(booted.runtime.content.projectDir).toBe(booted.contentDir);+ });+});diff --git a/packages/server/src/project-runtime.ts b/packages/server/src/project-runtime.tsnew file mode 100644index 000000000..213786a1e--- /dev/null+++ b/packages/server/src/project-runtime.ts@@ -0,0 +1,123 @@+import type { ServerInstance } from './server-factory.ts';++/**+ * The coarse-grained boundary around one project's stateful resources — the+ * composition surface the capability services (documents, files, search,+ * assets, git/sync, skills, config) build against instead of reaching into+ * constructor internals or hopping through the server's own HTTP API.+ *+ * Grain is deliberately coarse: themed groups of existing resources, no+ * fine-grained sub-interfaces. A seam is promoted to its own injection point+ * only when a second implementation exists to test it against — speculative+ * seam placement is itself a retrofit risk. Every local assumption+ * (filesystem paths, shadow-git, one-project-one-process) stays inside the+ * local constructor; a future runtime implementation varies behind this+ * interface, never by forking consumers.+ *+ * Deliberately NOT here, because they are transport concerns owned by the+ * HTTP composition layer: the HTTP server and port, route mounting, CORS and+ * admission policy, and `lockDir` — the lock file is how transport+ * advertises "a server is dialable at port N", and a runtime has no port.+ * `serverInstanceId` and `durabilityState` stay: their values are runtime+ * state (CRDT restart-recovery reads them); only their advertisement over+ * `/api/server-info` and CC1 belongs to transport.+ */+export interface ProjectRuntime {+ /** Collaborative-document engine and its restart-recovery state. */+ readonly crdt: {+ readonly hocuspocus: ServerInstance['hocuspocus'];+ readonly durabilityState: ServerInstance['durabilityState'];+ readonly cc1Broadcaster: ServerInstance['cc1Broadcaster'];+ readonly serverInstanceId: ServerInstance['serverInstanceId'];+ };+ /** Filesystem anchors and content admission. */+ readonly content: {+ readonly contentDir: string;+ /** Project anchor (`.ok/` home). Equals `contentDir` unless `content.dir` points elsewhere. */+ readonly projectDir: string;+ readonly contentFilter: ServerInstance['contentFilter'];+ };+ /** Derived, rebuildable lookup structures. */+ readonly indexes: {+ readonly basenameIndex: ServerInstance['basenameIndex'];+ readonly resolveEmbed: ServerInstance['resolveEmbed'];+ };+ /** Git-backed sync. Null when dormant or no remote is configured. */+ readonly git: {+ readonly syncEngine: ServerInstance['syncEngine'];+ };+ /** Agent sessions, presence, and ACP integration. */+ readonly agents: {+ readonly sessionManager: ServerInstance['sessionManager'];+ readonly agentFocusBroadcaster: ServerInstance['agentFocusBroadcaster'];+ readonly agentPresenceBroadcaster: ServerInstance['agentPresenceBroadcaster'];+ readonly acpRegistry: ServerInstance['acpRegistry'];+ readonly acpPermissions: ServerInstance['acpPermissions'];+ };+ /** Off-write-path background work. Absent in plugin/ephemeral modes. */+ readonly background: {+ readonly maintenanceCoordinator: ServerInstance['maintenanceCoordinator'];+ };+ /** Fresh-read configuration accessors. Grows as capability extraction promotes them. */+ readonly config: {+ readonly getLinkPreviewsEnabled: ServerInstance['getLinkPreviewsEnabled'];+ };+ readonly ready: ServerInstance['ready'];+ readonly degraded: ServerInstance['degraded'];+ readonly destroy: ServerInstance['destroy'];+}++/**+ * Group an existing `ServerInstance` into the runtime boundary. A pure view:+ * every member IS the instance's member (identity-preserved), so adopting the+ * runtime is a call-site change, never a behavior change. The path anchors+ * come from the composer because `createServer` never exposes them.+ */+export function createProjectRuntime(+ instance: ServerInstance,+ anchors: { contentDir: string; projectDir: string },+): ProjectRuntime {+ return {+ crdt: {+ hocuspocus: instance.hocuspocus,+ durabilityState: instance.durabilityState,+ cc1Broadcaster: instance.cc1Broadcaster,+ serverInstanceId: instance.serverInstanceId,+ },+ content: {+ contentDir: anchors.contentDir,+ projectDir: anchors.projectDir,+ contentFilter: instance.contentFilter,+ },+ indexes: {+ basenameIndex: instance.basenameIndex,+ resolveEmbed: instance.resolveEmbed,+ },+ git: {+ // Live getter, not a value copy: the engine is assigned after async+ // init, and `instance.syncEngine` is itself a getter over that slot.+ get syncEngine() {+ return instance.syncEngine;+ },+ },+ agents: {+ sessionManager: instance.sessionManager,+ agentFocusBroadcaster: instance.agentFocusBroadcaster,+ agentPresenceBroadcaster: instance.agentPresenceBroadcaster,+ acpRegistry: instance.acpRegistry,+ acpPermissions: instance.acpPermissions,+ },+ background: {+ maintenanceCoordinator: instance.maintenanceCoordinator,+ },+ config: {+ getLinkPreviewsEnabled: instance.getLinkPreviewsEnabled,+ },+ ready: instance.ready,+ // Getter for the same liveness reason: the list fills during init.+ get degraded() {+ return instance.degraded;+ },+ destroy: instance.destroy,+ };+}diff --git a/packages/server/src/services/asset-classification.ts b/packages/server/src/services/asset-classification.tsnew file mode 100644index 000000000..856d2b7ff--- /dev/null+++ b/packages/server/src/services/asset-classification.ts@@ -0,0 +1,123 @@+import { extname } from 'node:path';+import { SANDBOXED_HTML_CSP, SANDBOXED_HTML_EXTENSIONS } from '@inkeep/open-knowledge-core';+import { mimes } from 'mrmime';++/**+ * The one extension → disposition/CSP policy for serving content assets.+ * Both serve surfaces (`/api/asset` and the static content middleware) apply+ * this identically; before this module each reimplemented it by hand with a+ * "mirrors the other" comment as the only sync mechanism.+ *+ * `html`/`htm` render inline ONLY under the sandbox CSP — they are+ * deliberately absent from INLINE_RENDERABLE_EXTENSIONS so no branch serves+ * them as a plain same-origin document. SVG inline executes embedded scripts+ * on top-level navigation (`nosniff` doesn't help — `image/svg+xml` is+ * CORB-excluded), hence its own sandbox CSP; `<img>` embeds are unaffected.+ */+export interface AssetDisposition {+ disposition: 'inline' | 'attachment';+ /** Content-Security-Policy to apply, or null when none is required. */+ csp: string | null;+ /** True for the sandboxed-HTML class (callers add no-store cache policy). */+ sandboxedHtml: boolean;+}++export function classifyAssetDisposition(+ ext: string,+ /** Injectable for tests; production callers pass INLINE_RENDERABLE_EXTENSIONS. */+ inlineExtensions: ReadonlySet<string>,+): AssetDisposition {+ const sandboxedHtml = SANDBOXED_HTML_EXTENSIONS.has(ext);+ const disposition = inlineExtensions.has(ext) || sandboxedHtml ? 'inline' : 'attachment';+ const csp =+ ext === 'svg'+ ? "sandbox; default-src 'none'; style-src 'unsafe-inline'"+ : sandboxedHtml+ ? SANDBOXED_HTML_CSP+ : null;+ return { disposition, csp, sandboxedHtml };+}++/**+ * Close 3 gaps in mrmime's default mime table that break browser inline+ * rendering for common user-drop formats. Without these, sirv serves the+ * bytes with an empty `Content-Type` header — combined with our+ * `Content-Disposition: inline` policy, Chromium renders the binary+ * bytes as garbled text rather than dispatching to its built-in video /+ * audio viewer.+ *+ * The fix is documented idiomatic usage per mrmime's README: "Exposes+ * the `mimes` dictionary for easy additions or overrides." Three+ * extensions need coverage:+ *+ * - `.m4v` → `video/mp4`. Apple's MP4 variant is structurally MP4;+ * `video/mp4` is standards-recommended (WordPress Trac #24993,+ * Mozilla bug 875573). mrmime deliberately filters `x-` types, so+ * the historical `video/x-m4v` is not in its default table.+ * - `.mkv` → `video/x-matroska`. De-facto type (no IANA registration+ * exists); Chromium recognizes it. Only non-`x-` alternative would+ * be `application/octet-stream` which blocks inline rendering.+ * - `.flac` → `audio/flac`. IANA-registered (RFC 9639);+ * `audio/x-flac` is the deprecated legacy alias.+ *+ * Security posture: setting extension-derived Content-Type on+ * video/audio with `X-Content-Type-Options: nosniff` is NOT a stored-+ * XSS vector. Browsers refuse to treat `video/*` / `audio/*` as+ * scriptable regardless of file contents under nosniff (MDN+ * X-Content-Type-Options, Beyond XSS ch5). The SVG polyglot class+ * (`image/svg+xml`) is the real risk and is separately covered by+ * `EXECUTABLE_BLOCKLIST_EXTENSIONS` barring `.svg` from the+ * `openAssetSafely` click path.+ *+ * Module-load mutation runs once per Node process. Multiple dev-server+ * invocations in the same process (Vite restart) re-assign idempotently.+ *+ * If a future inline-renderable extension lands without a mrmime entry,+ * the narrow-integration test for `.m4v` will flag it (currently pinned+ * to `video/mp4`). Extend this map in lockstep.+ */+Object.assign(mimes, {+ m4v: 'video/mp4',+ mkv: 'video/x-matroska',+ flac: 'audio/flac',+ // TOML has an IANA-registered media type (`application/toml`) but+ // `mrmime` doesn't ship it by default — the table is the+ // narrow `mime-db` subset, not the full registry. Without this+ // entry, sirv serves `.toml` with an empty `Content-Type` and our+ // `/api/asset` handler 415s (the `assetContentTypeForPath` lookup+ // returns null). The `TextViewer`'s own fetch path+ // (`/api/asset-text`) forces `text/plain` and is therefore+ // unaffected by this patch — what relies on it is the fallback+ // pane's "Open file" link + any direct deeplink to a `.toml`+ // asset URL. JSON is already covered by mrmime's defaults.+ toml: 'application/toml',+ // `.lock` has no IANA registration and no mrmime default. Same+ // mrmime-gap pattern as `.toml` above; without this, the+ // `INLINE_RENDERABLE_EXTENSIONS` widening for `lock` would 415 on+ // direct `/api/asset?path=foo.lock` GETs. `text/plain` matches+ // what the `TextViewer` path (`/api/asset-text`) already forces,+ // so the sidebar-click and deeplink surfaces agree on the wire+ // shape. Lockfile contents vary across ecosystems (some JSON-+ // shaped, some custom DSLs) but `text/plain` is the right floor.+ lock: 'text/plain',+ // Attachment-only types newly admitted to ASSET_EXTENSIONS that `mrmime`'s+ // default table omits. Without an entry, `assetContentTypeForPath` returns+ // null and `handleAsset` 415s them (and sirv streams them with an empty+ // Content-Type). All download-only — none are inline-renderable.+ '7z': 'application/x-7z-compressed',+ tar: 'application/x-tar',+ rar: 'application/vnd.rar',+ xls: 'application/vnd.ms-excel',+ ppt: 'application/vnd.ms-powerpoint',+ odt: 'application/vnd.oasis.opendocument.text',+ ods: 'application/vnd.oasis.opendocument.spreadsheet',+ odp: 'application/vnd.oasis.opendocument.presentation',+ pages: 'application/vnd.apple.pages',+ numbers: 'application/vnd.apple.numbers',+ key: 'application/vnd.apple.keynote',+ mobi: 'application/x-mobipocket-ebook',+});++export function assetContentTypeForPath(path: string): string | null {+ return mimes[extname(path).slice(1).toLowerCase()] ?? null;+}diff --git a/packages/server/src/services/assets.ts b/packages/server/src/services/assets.tsnew file mode 100644index 000000000..037bd648c--- /dev/null+++ b/packages/server/src/services/assets.ts@@ -0,0 +1,558 @@+import { createHash } from 'node:crypto';+import {+ closeSync,+ createReadStream,+ existsSync,+ openSync,+ readSync,+ realpathSync,+ statSync,+ unlinkSync,+} from 'node:fs';+import { readdir, stat } from 'node:fs/promises';+import { dirname, extname, relative, resolve } from 'node:path';+import { pipeline } from 'node:stream/promises';+import {+ ASSET_EXTENSIONS,+ DEFAULT_ATTACHMENT_FOLDER_PATH,+ DEFAULT_DEDUP_MODE,+ INLINE_RENDERABLE_EXTENSIONS,+ isValidAttachmentFolderPath,+ normalizeAttachmentFolderPath,+} from '@inkeep/open-knowledge-core';+import { fileTypeFromBuffer } from 'file-type';+import { toContentRelativePath } from '../asset-references.ts';+import { isWithinContentDir } from '../content-path.ts';+import { sanitizeFilename } from '../filename-sanitize.ts';+import { assertNoSymlinkEscape, isAlreadyExistsError } from '../fs-safety.ts';+import { tracedMkdirSync } from '../fs-traced.ts';+import { errnoCode } from '../http/handler-utils.ts';+import { getLogger } from '../logger.ts';+import { toPosix } from '../path-utils.ts';+import { classifyUploadErrno, UploadWriteError, type UploadWriteReason } from '../upload-errors.ts';+import { linkTempToFinalWithCollisionRetry } from '../upload-streaming.ts';+import {+ type AssetDisposition,+ assetContentTypeForPath,+ classifyAssetDisposition,+} from './asset-classification.ts';++/**+ * Domain operations for the assets capability. Transports (HTTP handlers,+ * MCP tools) parse their protocol, call these, and render the outcome;+ * every filesystem decision — canonicalization, containment, existence,+ * classification, and the upload write path — lives here exactly once.+ *+ * Outcome `reason`/`kind` values are transport-neutral; the HTTP handlers+ * own the mapping to status/URN/title (which the characterization tests pin).+ */++interface ServableAsset extends AssetDisposition {+ canonicalPath: string;+ relativePath: string;+ size: number;+ contentType: string;+}++type ServeResolution =+ | { ok: true; asset: ServableAsset }+ | {+ ok: false;+ reason: 'missing-path' | 'unsupported-type' | 'not-found' | 'invalid-path';+ /** Underlying fs error, when one produced the failure — log-only, never on the wire. */+ cause?: unknown;+ };++type TextResolution =+ | { ok: true; canonicalPath: string; size: number }+ | { ok: false; reason: 'missing-path' | 'not-found' | 'invalid-path'; cause?: unknown };++interface StoreUploadInput {+ /** Streamed multipart tempfile (the transport's `readUploadBody` output). */+ tempPath: string;+ /** sha256 hex of the tempfile bytes, from the streaming hash pass. */+ sha: string;+ byteLength: number;+ /** Client-supplied filename; may be a generic clipboard name. */+ filename: string;+ /** Schema-validated, escape-checked by the transport before the call. */+ parentDocName: string;+ placement: 'configured-attachments' | 'parent-dir' | undefined;+}++type StoreUploadOutcome =+ | { ok: true; src: string; path: string; deduped: boolean; mime: string | null }+ | { ok: false; kind: 'config-error'; cause: unknown }+ | { ok: false; kind: 'invalid-attachment-folder' }+ | { ok: false; kind: 'path-escape'; cause?: unknown }+ | { ok: false; kind: 'dest-validation-error'; cause: unknown; destDir: string }+ | { ok: false; kind: 'mkdir-failed'; reason: UploadWriteReason; cause: unknown }+ | {+ ok: false;+ kind: 'write-failed';+ reason: UploadWriteReason;+ cause: unknown;+ filename: string;+ };++export interface AssetService {+ /** `/api/asset` domain: extension-gated, ignore-filtered, classified for serving. */+ resolveServableAsset(requestedPath: string | null): ServeResolution;+ /**+ * `/api/asset-text` domain: same containment core, deliberately NO+ * extension gate and NO ignore filter — the caller reaches this only for+ * files it can already see, and blocking here would break the+ * read-a-hidden-file workflow. Path safety stays load-bearing.+ */+ resolveTextAsset(requestedPath: string | null): TextResolution;+ /**+ * `/api/upload` write path: attachment-folder resolution, containment and+ * symlink-escape enforcement around directory creation, magic-byte sniff,+ * same-dir sha256 dedup, filename synthesis, and the tempfile→final link.+ * Owns the tempfile: every failure path unlinks it; success consumes it.+ */+ storeUpload(input: StoreUploadInput): Promise<StoreUploadOutcome>;+}++export interface AssetServiceDeps {+ contentDir: string;+ /** Ignore filter for the servable path; text resolution never consults it. */+ isPathIgnored?: (relativePath: string) => boolean;+ /** Fresh-read `content.attachmentFolderPath` accessor; may throw on invalid config. */+ getAttachmentFolderPath?: () => string;+}++type CanonicalResolution =+ | { ok: true; canonicalPath: string; relativePath: string; size: number }+ | { ok: false; reason: 'not-found' | 'invalid-path'; cause?: unknown };++const GENERIC_PASTE_NAMES = /^(image\.(png|jpe?g|gif|webp)|Clipboard.*|Untitled.*)$/i;++/**+ * Resolve the destination directory for an upload from the parent doc's+ * path and the configured `content.attachmentFolderPath`. Matches Obsidian's+ * literal schema (free-form string):+ *+ * - `"./"` (default) → same directory as the doc+ * - `"/"` → content-directory root+ * - `"./<sub>"` → subdirectory beside the doc+ * - `"<name>"` (bare) → fixed content-relative path+ *+ * Treats any `./` prefix as "relative to doc dir," any other value as+ * "relative to content dir." Empty or whitespace-only strings fall back+ * to the default (doc dir).+ *+ * Returns an absolute path within `resolvedContentDir` — path-escape+ * enforcement happens at the caller via `isWithinContentDir` + `realpath`.+ */+export function resolveUploadDestDir(+ parentDocName: string,+ attachmentFolderPath: string,+ resolvedContentDir: string,+): string {+ const trimmed = attachmentFolderPath.trim();+ if (trimmed === '' || trimmed === './') {+ return resolve(resolvedContentDir, dirname(parentDocName));+ }+ if (trimmed === '/') {+ return resolvedContentDir;+ }+ if (trimmed.startsWith('./')) {+ // Subdirectory beside the doc. `"./attachments"` → `<docDir>/attachments`.+ return resolve(resolvedContentDir, dirname(parentDocName), trimmed.slice(2));+ }+ // Bare name or nested path: fixed content-relative location.+ return resolve(resolvedContentDir, trimmed);+}++/**+ * Read at most `n` bytes from the start of `path`. Feeds both the magic-byte+ * sniff (`fileTypeFromBuffer` over the head) and the SVG text fallback+ * (`file-type` can't detect text-based SVG), without ever materializing the+ * whole file.+ */+function readTempFileHead(path: string, n: number): Buffer {+ const fd = openSync(path, 'r');+ try {+ const buf = Buffer.alloc(n);+ const read = readSync(fd, buf, 0, n, 0);+ return buf.subarray(0, read);+ } finally {+ closeSync(fd);+ }+}++/**+ * Upper bound on size-matched candidates we'll read+hash in a single+ * dedup call. A capture-device folder with 1000+ screenshots at the same+ * resolution could theoretically produce that many same-size siblings;+ * each candidate costs a sync readFileSync + sha256Hex of the entire+ * buffer, which would block the event loop for seconds per upload under+ * adversarial / pathological load.+ *+ * Past the bound, dedup degrades to best-effort: we log a structured+ * WARN and return null (treat as no-match → write a new file with the+ * collision-suffix loop). This is a bounded-resource defense, not a+ * correctness change — a duplicate that slips through produces the+ * cheap storage cost of one extra on-disk copy, not silent data loss.+ * The O(1) hash-cache alternative is a+ * larger architectural change and a follow-on.+ */+const MAX_DEDUP_SCAN_CANDIDATES = 1000;++/**+ * Stream a file's bytes through a sha256 Hash transform and return the hex+ * digest. Keeps memory O(1) regardless of file size — a 500 MB candidate+ * read by the buffer-based `readFileSync` path would otherwise materialize+ * the whole file in heap, which defeats the streaming-upload amendment's+ * O(1) memory guarantee.+ *+ * Throws on read errors so the caller can classify ENOENT (concurrent+ * rename — stay silent) vs other errors (log and skip).+ */+async function streamingHashFile(path: string): Promise<string> {+ const hash = createHash('sha256');+ await pipeline(createReadStream(path), hash);+ return hash.digest('hex');+}++/**+ * Scan `destDir` non-recursively for an existing file whose sha256 matches+ * the buffer's. Returns the matching basename (case-preserving) or null if+ * no match. Bounded by directory size — O(n) in sibling count, not vault size.+ * Only files with extensions in ASSET_EXTENSIONS are candidates; everything+ * else (markdown, .git/, etc.) is skipped.+ *+ * `expectedSize` is the buffer's byte length — passed in so we can size-+ * prefilter before hashing siblings. sha256 collision requires equal-sized+ * inputs, so same-extension siblings with a different size are not+ * candidates and we skip their (potentially multi-MB) read.+ */+async function findDuplicateAsset(+ destDir: string,+ sha: string,+ expectedSize: number,+): Promise<string | null> {+ let entries: string[];+ try {+ // Async `readdir` so the directory walk doesn't block the event+ // loop during uploads — Node's event loop is shared with WebSocket sync+ // and CRDT updates, and a 1k-entry walk is observable on bursty+ // upload traffic. The MAX_DEDUP_SCAN_CANDIDATES cap+ // bounds the worst case at 1000 same-size siblings, but the+ // pre-cap entry list can still be much larger.+ entries = await readdir(destDir);+ } catch {+ return null;+ }+ const log = getLogger('upload');+ let scanned = 0;+ for (const entry of entries) {+ const ext = extname(entry).slice(1).toLowerCase();+ if (!ASSET_EXTENSIONS.has(ext)) continue;+ const fullPath = resolve(destDir, entry);+ let entryStat: Awaited<ReturnType<typeof stat>>;+ try {+ entryStat = await stat(fullPath);+ } catch {+ continue;+ }+ if (!entryStat.isFile() || entryStat.size !== expectedSize) continue;+ // Bounded scan: only count candidates that passed the cheap size+ // prefilter, since same-size siblings are the ones that cost a+ // full-file hash each (streaming now, not buffered).+ scanned++;+ if (scanned > MAX_DEDUP_SCAN_CANDIDATES) {+ log.warn(+ {+ event: 'upload-dedup-skip',+ reason: 'scan-cap-exceeded',+ destDir,+ scanned: MAX_DEDUP_SCAN_CANDIDATES,+ expectedSize,+ },+ `[upload-dedup] candidate scan exceeded ${MAX_DEDUP_SCAN_CANDIDATES} same-size siblings — degrading to no-dedup for this upload`,+ );+ return null;+ }+ let candidateSha: string;+ try {+ // Stream + hash the candidate to preserve the O(1) memory guarantee+ // the upload pipeline otherwise maintains end-to-end. A 500 MB+ // candidate otherwise spiked heap to 500 MB per scan.+ candidateSha = await streamingHashFile(fullPath);+ } catch (err) {+ const code = errnoCode(err);+ // ENOENT is the legitimate concurrent-rename race — stay silent.+ if (code !== 'ENOENT') {+ log.warn(+ { event: 'upload-dedup-skip', reason: 'read-failed', code, entry },+ '[upload-dedup] skipped candidate — read failed',+ );+ }+ continue;+ }+ if (candidateSha === sha) return entry;+ }+ return null;+}++export function createAssetService(deps: AssetServiceDeps): AssetService {+ /**+ * The shared containment core: resolve against the realpath'd content+ * root, canonicalize, verify containment, require a regular file, and+ * reject any spelling that is not already canonical (traversal segments,+ * backslashes) even when it stays inside the root.+ */+ function resolveCanonical(assetPath: string): CanonicalResolution {+ const resolvedContentDir = realpathSync(deps.contentDir);+ const requestedPath = resolve(resolvedContentDir, assetPath);+ let canonicalPath: string;+ try {+ canonicalPath = realpathSync(requestedPath);+ } catch (err) {+ return { ok: false, reason: 'not-found', cause: err };+ }+ if (!isWithinContentDir(canonicalPath, resolvedContentDir)) {+ return { ok: false, reason: 'invalid-path' };+ }+ let stat: ReturnType<typeof statSync>;+ try {+ stat = statSync(canonicalPath);+ } catch (err) {+ return { ok: false, reason: 'not-found', cause: err };+ }+ if (!stat.isFile()) {+ return { ok: false, reason: 'not-found' };+ }+ const relativePath = toContentRelativePath(resolvedContentDir, canonicalPath);+ if (relativePath !== assetPath.split('\\').join('/')) {+ return { ok: false, reason: 'invalid-path' };+ }+ return { ok: true, canonicalPath, relativePath, size: stat.size };+ }++ async function storeUpload(input: StoreUploadInput): Promise<StoreUploadOutcome> {+ const { tempPath, sha, byteLength, filename, parentDocName, placement } = input;++ // Belt-and-braces cleanup: every failure path below that does NOT+ // consume the tempfile via linkTempToFinal* runs this.+ const cleanupTempfile = () => {+ if (existsSync(tempPath)) {+ try {+ unlinkSync(tempPath);+ } catch {+ // best-effort; orphan sweep reaps stragglers+ }+ }+ };++ const resolvedContentDir = resolve(deps.contentDir);+ let rawAttachmentFolderPath: string;+ try {+ rawAttachmentFolderPath =+ placement === 'parent-dir'+ ? DEFAULT_ATTACHMENT_FOLDER_PATH+ : (deps.getAttachmentFolderPath?.() ?? DEFAULT_ATTACHMENT_FOLDER_PATH);+ } catch (err) {+ cleanupTempfile();+ return { ok: false, kind: 'config-error', cause: err };+ }+ if (!isValidAttachmentFolderPath(rawAttachmentFolderPath)) {+ cleanupTempfile();+ return { ok: false, kind: 'invalid-attachment-folder' };+ }+ const attachmentFolderPath = normalizeAttachmentFolderPath(rawAttachmentFolderPath);+ const destDir = resolveUploadDestDir(parentDocName, attachmentFolderPath, resolvedContentDir);+ if (!isWithinContentDir(destDir, resolvedContentDir)) {+ cleanupTempfile();+ return { ok: false, kind: 'path-escape' };+ }+ // Pre-mkdir symlink-escape check: walks up from destDir to the+ // deepest existing ancestor and rejects if its realpath escapes contentDir.+ // Doing this before `tracedMkdirSync({ recursive: true })` prevents mkdir from+ // following a parent symlink and materializing a fresh directory outside+ // contentDir. The post-mkdir realpath check below remains as defense-in-+ // depth against TOCTOU symlink-replace races between this check and mkdir.+ try {+ assertNoSymlinkEscape(destDir, resolvedContentDir);+ } catch (err) {+ cleanupTempfile();+ const message = err instanceof Error ? err.message : String(err);+ if (message.startsWith('symlink-escape:')) {+ return { ok: false, kind: 'path-escape' };+ }+ return { ok: false, kind: 'dest-validation-error', cause: err, destDir };+ }+ // mkdir -p the destination — bare-name / nested attachmentFolderPath+ // values produce directories that may not exist at first upload.+ try {+ tracedMkdirSync(destDir, { recursive: true });+ } catch (err) {+ if (!isAlreadyExistsError(err)) {+ cleanupTempfile();+ // Classify the errno through the same typed table the streaming-+ // write path uses so ENOSPC/EDQUOT route through 507 storage-full+ // and EROFS/EACCES/EPERM route through 500 storage-readonly —+ // SDK consumers branch on the URN, not the errno.+ const reason = classifyUploadErrno(err as NodeJS.ErrnoException);+ return { ok: false, kind: 'mkdir-failed', reason, cause: err };+ }+ }++ // Symlink escape check: realpath the dest dir and compare against realpath'd contentDir+ try {+ const realDestDir = realpathSync(destDir);+ let realContentDir: string;+ try {+ realContentDir = realpathSync(resolvedContentDir);+ } catch {+ realContentDir = resolvedContentDir;+ }+ if (!isWithinContentDir(realDestDir, realContentDir)) {+ cleanupTempfile();+ return { ok: false, kind: 'path-escape' };+ }+ } catch (e) {+ const code = errnoCode(e);+ if (code === 'ENOENT') {+ // Directory doesn't exist yet — will be created below; no symlink escape possible+ } else {+ cleanupTempfile();+ return { ok: false, kind: 'path-escape', cause: e };+ }+ }++ // Accept-all: every file is accepted — there's no user-facing byte cap+ // post-streaming (disk fullness surfaces as 507 instead). The magic-+ // byte sniff is only consulted to (a) preserve the SVG `<img>`-only+ // routing for security and (b) recover an extension when the upload+ // arrived with a generic clipboard filename. Non-sniffable bytes are+ // accepted under the client-supplied filename.+ const SNIFF_HEAD_BYTES = 4100;+ const head = readTempFileHead(tempPath, SNIFF_HEAD_BYTES);+ const fileTypeResult = await fileTypeFromBuffer(head);+ let detectedMime: string | undefined = fileTypeResult?.mime;+ let detectedExt: string | undefined = fileTypeResult?.ext;+ // file-type can't detect SVG (text-based, no magic bytes) — check manually.+ // STOP: this fallback is LOAD-BEARING — SVG must render via+ // <img>, never inline DOM. Do not remove without a compensating guard.+ if (!detectedMime) {+ // Strip a leading UTF-8 BOM (U+FEFF) before the pattern match.+ // `trimStart()` removes ECMAScript whitespace but not the BOM, so a+ // file starting with `\xEF\xBB\xBF<svg ...>` would otherwise evade the+ // head check the comment above documents as the SVG-disguised-as-PNG+ const headText = head.subarray(0, 256).toString('utf-8').replace(/^/, '').trimStart();+ if (+ headText.startsWith('<svg') ||+ (headText.startsWith('<?xml') && headText.includes('<svg'))+ ) {+ detectedMime = 'image/svg+xml';+ detectedExt = 'svg';+ }+ }++ // Same-dir sha256 dedup. Bounded scan over destDir, skipped entirely+ // when DEFAULT_DEDUP_MODE === 'off'. The dedup test happens BEFORE+ // filename synthesis so a duplicate paste preserves the existing+ // on-disk basename instead of producing a fresh pasted-<ts>.png stub.+ //+ // On a dedup hit the tempfile is unlinked and we short-circuit without+ // touching the destDir inode — `linkTempToFinalWithCollisionRetry`+ // never runs.+ if (DEFAULT_DEDUP_MODE === 'same-dir') {+ const existing = await findDuplicateAsset(destDir, sha, byteLength);+ if (existing) {+ cleanupTempfile();+ const relPath = toPosix(relative(deps.contentDir, resolve(destDir, existing)));+ return {+ ok: true,+ src: existing,+ path: relPath,+ deduped: true,+ mime: detectedMime ?? null,+ };+ }+ }++ // GENERIC_PASTE_NAMES: clipboard paste arrives with synthetic names+ // ("image.png", "Clipboard 2024-04-21 14:23:45"). Replace with a+ // timestamp stem so the disk filename is human-meaningful.+ let finalFilename: string;+ const isGenericPaste = !filename || filename === 'upload' || GENERIC_PASTE_NAMES.test(filename);+ if (isGenericPaste) {+ const now = new Date();+ const ts = now+ .toISOString()+ .replace(/[-:T]/g, '')+ .slice(0, 14)+ .replace(/(\d{8})(\d{6})/, '$1-$2');+ // Prefer the sniffed extension when present; otherwise try the+ // client-supplied extname, finally fall back to .bin.+ const fallbackExt = filename ? extname(filename).slice(1) : '';+ const ext = detectedExt ?? fallbackExt ?? '';+ finalFilename = ext === '' ? `pasted-${ts}` : `pasted-${ts}.${ext}`;+ } else {+ finalFilename = sanitizeFilename(filename);+ }++ try {+ const destFilename = linkTempToFinalWithCollisionRetry(tempPath, destDir, finalFilename);+ const relPath = toPosix(relative(deps.contentDir, resolve(destDir, destFilename)));+ return {+ ok: true,+ src: destFilename,+ path: relPath,+ deduped: false,+ mime: detectedMime ?? null,+ };+ } catch (e) {+ // linkTempToFinalWithCollisionRetry best-effort unlinks the tempfile+ // on throw; no extra cleanupTempfile() call needed here.+ const reason: UploadWriteReason =+ e instanceof UploadWriteError ? e.reason : 'urn:ok:error:storage-error';+ return { ok: false, kind: 'write-failed', reason, cause: e, filename: finalFilename };+ }+ }++ return {+ resolveServableAsset(requestedPath) {+ if (!requestedPath || requestedPath.includes('\0')) {+ return { ok: false, reason: 'missing-path' };+ }+ const contentType = assetContentTypeForPath(requestedPath);+ const ext = extname(requestedPath).slice(1).toLowerCase();+ if (!contentType || !ASSET_EXTENSIONS.has(ext)) {+ return { ok: false, reason: 'unsupported-type' };+ }+ const core = resolveCanonical(requestedPath);+ if (!core.ok) return core;+ // Ignore-filter AFTER containment, reported as not-found: exclusion is+ // opaque on the wire, identical to a missing file.+ if (deps.isPathIgnored?.(core.relativePath)) {+ return { ok: false, reason: 'not-found' };+ }+ return {+ ok: true,+ asset: {+ canonicalPath: core.canonicalPath,+ relativePath: core.relativePath,+ size: core.size,+ contentType,+ ...classifyAssetDisposition(ext, INLINE_RENDERABLE_EXTENSIONS),+ },+ };+ },++ resolveTextAsset(requestedPath) {+ if (!requestedPath || requestedPath.includes('\0')) {+ return { ok: false, reason: 'missing-path' };+ }+ const core = resolveCanonical(requestedPath);+ if (!core.ok) return core;+ return { ok: true, canonicalPath: core.canonicalPath, size: core.size };+ },++ storeUpload,+ };+}diff --git a/packages/server/src/services/file-ops.ts b/packages/server/src/services/file-ops.tsnew file mode 100644index 000000000..9ca2e7c1b--- /dev/null+++ b/packages/server/src/services/file-ops.ts@@ -0,0 +1,532 @@+import { existsSync, readFileSync, statSync } from 'node:fs';+import { dirname, extname, resolve } from 'node:path';+import { isConfigDoc, isSystemDoc } from '../cc1-broadcast.ts';+import type { DerivedDocumentIndexMutation } from '../derived-document-index.ts';+import { forgetDocExtension, registerDocExtension, stripDocExtension } from '../doc-extensions.ts';+import { contentHash, registerWrite } from '../file-watcher.ts';+import { isAlreadyExistsError } from '../fs-safety.ts';+import {+ tracedCpSync,+ tracedMkdirSync,+ tracedRmdirSync,+ tracedRmSync,+ tracedUnlinkSync,+ tracedWriteFileSync,+} from '../fs-traced.ts';+import { errnoCode } from '../http/handler-utils.ts';+import { getLogger } from '../logger.ts';++/**+ * Domain operations for the files/folders capability (create, hard delete,+ * trash reconciliation, duplication). Transports validate and map outcomes;+ * this module owns the ordered server-state teardown that+ * both destructive paths share:+ *+ * capture-and-close live docs → mark recently-removed → (delete only:+ * disk mutation) → index purges → derived-doc deletion → CC1 signal+ *+ * Hard delete and trash-cleanup are deliberate twins: trash's file is+ * already in the OS Trash before the call (Step 1 of the two-step flow),+ * so it reconciles the same server state while never touching disk.+ *+ * Every deletion enumerates docs from DISK, not the lagging file index+ * (`listManagedDocNamesUnderFolder` via deps) — except trash-cleanup, whose+ * source of truth is deliberately the in-memory index: disk is already+ * post-deletion there, and an empty index means the watcher won the race+ * and the idempotent fast-path returns an empty list.+ */++type FileOpKind = 'file' | 'folder' | 'asset';++type CreateFolderOutcome = { ok: true } | { ok: false; kind: 'already-exists' };++type CreatePageOutcome = { ok: true } | { ok: false; kind: 'already-exists'; cause: unknown };++type DeletePathOutcome =+ | { ok: true; deletedDocNames: string[] }+ | { ok: false; kind: 'not-found' }+ | { ok: false; kind: 'type-mismatch' }+ | { ok: false; kind: 'conflict'; file: string };++type DuplicatePathOutcome =+ | { ok: true; duplicatedPath: string; duplicatedDocNames: string[] }+ | { ok: false; kind: 'not-found' }+ | { ok: false; kind: 'type-mismatch' }+ | { ok: false; kind: 'conflict'; file: string }+ | { ok: false; kind: 'destination-excluded' }+ | { ok: false; kind: 'already-exists'; cause: unknown };++export interface FileOpsDeps {+ contentDir: string;+ resolveContentEntryPath: (contentDir: string, kind: 'file' | 'folder', path: string) => string;+ docNameForPath: (relPath: string) => string;+ docNameToRelativePath: (docName: string) => string;+ listManagedDocNamesUnderFolder: (absFolderPath: string) => string[];+ listAffectedDocNames: (+ index: ReadonlyMap<string, unknown>,+ kind: 'folder',+ path: string,+ ) => string[];+ getFileIndex: () => ReadonlyMap<string, unknown>;+ /** Conflicted files per the sync engine's store (empty set when no engine). */+ getConflictedFiles: () => ReadonlySet<string>;+ /** Lifecycle-conflict check against the live Hocuspocus doc, when loaded. */+ isDocNameInLifecycleConflict: (docName: string) => boolean;+ captureAndCloseDocuments: (docNames: string[], reason: 'deleted-upstream') => Promise<unknown>;+ /** Present in project mode; absent in harnesses without the LRU. */+ markRecentlyRemoved?: (docName: string) => void;+ mutateFileIndexDelete?: (args: { path: string; docName: string }) => void;+ removeFolderIndexEntries: (folderPath: string) => void;+ upsertFolderIndexPathSegments: (folderPath: string) => void;+ deleteDerivedDocumentsBestEffort: (docNames: string[], source: string) => Promise<void>;+ invalidateReferencedAssetsCache: () => void;+ signalFiles: () => void;+ /** "name copy N" allocation; throws when all slots are taken. */+ nextAvailableDuplicateDocName: (sourceDocName: string) => { docName: string };+ nextAvailableDuplicateFolderPath: (sourceFolderPath: string) => { folderPath: string };+ /** Containment- and symlink-checked absolute destination for a doc duplicate. */+ resolveDuplicateDocPath: (docName: string, extension: string) => string;+ collectMarkdownCopies: (+ folderPath: string,+ ) => Array<{ docName: string; fullPath: string; content: string }>;+ collectFolderPaths: (folderPath: string) => string[];+ contentFilter?: {+ isExcluded(relativePath: string): boolean;+ isDirExcluded(relativePath: string): boolean;+ incrementMdDir(dir: string): void;+ decrementMdDir(dir: string): void;+ };+ unmarkRecentlyRemoved?: (docName: string) => void;+ mutateFileIndexCreate?: (args: { path: string; docName: string; content: string }) => void;+ recordDerivedDocumentBestEffort: (+ documentName: string,+ markdown: string,+ reason: string,+ ) => Promise<void>;+ recordDerivedMutationsBestEffort: (+ mutations: readonly DerivedDocumentIndexMutation[],+ reason: string,+ ) => Promise<void>;+}++export interface FileOpsService {+ createFolder(folderPath: string): CreateFolderOutcome;+ /**+ * Create-exclusive doc write plus the synchronous registration set that+ * keeps the watcher, content filter, and file index coherent with a+ * self-write. Deliberately synchronous end-to-end: the caller records+ * contributors immediately after, and any async yield before that+ * recording opens a window where a pending shadow-commit timer drains the+ * contributor accumulator without this file's attribution. The caller+ * then seeds the derived index and emits the CC1 signal.+ */+ createPage(input: {+ fullPath: string;+ docName: string;+ initialContent: string;+ }): CreatePageOutcome;+ deletePath(operationKind: FileOpKind, operationPath: string): Promise<DeletePathOutcome>;+ trashCleanup(+ operationKind: FileOpKind,+ path: string,+ operationDocName: string,+ logSource: string,+ ): Promise<{ deletedDocNames: string[] }>;+ /**+ * Unlike delete/trash, duplication does NOT emit the CC1 files signal —+ * the caller signals after contributor recording so attribution is+ * queryable by the time the renderer refetches the tree.+ */+ duplicatePath(+ kind: 'file' | 'folder',+ requestedPath: string,+ requestedDocName: string,+ ): Promise<DuplicatePathOutcome>;+}++const log = getLogger('file-ops');++export function createFileOpsService(deps: FileOpsDeps): FileOpsService {+ const { contentDir } = deps;++ function markRemoved(docNames: string[], logSource: string): void {+ if (!deps.markRecentlyRemoved) return;+ for (const docName of docNames) {+ // STOP gate at the documentName-keyed entry point: synthetic docs+ // cannot appear here today (path validation rejects them upstream),+ // but the filter stays defense-in-depth.+ if (isSystemDoc(docName) || isConfigDoc(docName)) continue;+ deps.markRecentlyRemoved(docName);+ console.info(+ JSON.stringify({+ event: 'recently-removed-docs-populate',+ docName,+ kind: 'deleted',+ source: logSource,+ }),+ );+ }+ }++ function purgeFileIndex(docNames: string[]): void {+ for (const docName of docNames) {+ deps.mutateFileIndexDelete?.({+ path: resolve(contentDir, deps.docNameToRelativePath(docName)),+ docName,+ });+ }+ }++ // Conflict-aware refusal, dual-source (same rationale as the rename+ // spine): the live doc's lifecycle state catches loaded docs, the sync+ // engine's store catches docs evicted from memory.+ function findConflictedFile(docNames: string[]): string | null {+ const conflictedFiles = deps.getConflictedFiles();+ for (const docName of docNames) {+ const filePath = deps.docNameToRelativePath(docName);+ if (deps.isDocNameInLifecycleConflict(docName) || conflictedFiles.has(filePath)) {+ return filePath;+ }+ }+ return null;+ }++ return {+ createPage(input) {+ const { fullPath, docName, initialContent } = input;+ tracedMkdirSync(dirname(fullPath), { recursive: true });+ try {+ tracedWriteFileSync(fullPath, initialContent, { encoding: 'utf-8', flag: 'wx' });+ } catch (err) {+ if (isAlreadyExistsError(err)) {+ return { ok: false, kind: 'already-exists', cause: err };+ }+ throw err;+ }+ // Eager invalidation: legitimate recreation at a recently-renamed or+ // recently-deleted name drops the stale cache entry so the next+ // connection admits cleanly. No-op when absent.+ deps.unmarkRecentlyRemoved?.(docName);+ // Synchronously bump the content filter's sibling-asset dirCount so any+ // sibling asset drop that follows is admitted by the `LINKABLE_ASSET_EXTENSIONS`+ // rule. The file watcher's `create` event will also increment later,+ // which would double-count — so we also `registerWrite` to mark this+ // as a self-write, and the watcher skips its own `incrementMdDir` on+ // self-writes. See file-watcher.ts for the paired logic.+ deps.contentFilter?.incrementMdDir(dirname(docName));+ registerWrite(fullPath, contentHash(initialContent));+ deps.mutateFileIndexCreate?.({ path: fullPath, docName, content: initialContent });+ return { ok: true };+ },++ createFolder(folderPath) {+ const fullPath = deps.resolveContentEntryPath(contentDir, 'folder', folderPath);+ if (existsSync(fullPath)) {+ return { ok: false, kind: 'already-exists' };+ }+ tracedMkdirSync(fullPath, { recursive: true });+ deps.upsertFolderIndexPathSegments(folderPath);+ deps.signalFiles();+ return { ok: true };+ },++ async deletePath(operationKind, operationPath) {+ const targetPath =+ operationKind === 'asset'+ ? deps.resolveContentEntryPath(contentDir, 'folder', operationPath)+ : deps.resolveContentEntryPath(contentDir, operationKind, operationPath);+ if (!existsSync(targetPath)) {+ return { ok: false, kind: 'not-found' };+ }+ const targetStat = statSync(targetPath);+ if (+ ((operationKind === 'file' || operationKind === 'asset') && !targetStat.isFile()) ||+ (operationKind === 'folder' && !targetStat.isDirectory())+ ) {+ return { ok: false, kind: 'type-mismatch' };+ }++ // Disk-truth enumeration BEFORE the disk delete — the watcher-fed+ // index lags fresh writes, and an empty list here would skip the+ // capture/close + recently-removed population while the rm below+ // still removes the directory (orphaned in-memory Y.Docs).+ const deletedDocNames =+ operationKind === 'asset'+ ? []+ : operationKind === 'file'+ ? [deps.docNameForPath(operationPath)]+ : deps.listManagedDocNamesUnderFolder(+ deps.resolveContentEntryPath(contentDir, 'folder', operationPath),+ );++ // Any conflicted child blocks a folder delete — resolution must+ // finish first.+ const conflictedFile = findConflictedFile(deletedDocNames);+ if (conflictedFile !== null) {+ return { ok: false, kind: 'conflict', file: conflictedFile };+ }++ await deps.captureAndCloseDocuments(deletedDocNames, 'deleted-upstream');+ // Populate the removal cache BEFORE the disk delete so a fast+ // reconnect that observes the file gone via the watcher also sees+ // the cache entry.+ markRemoved(deletedDocNames, 'handleDeletePath');++ if (operationKind === 'file' || operationKind === 'asset') {+ tracedUnlinkSync(targetPath);+ } else {+ tracedRmSync(targetPath, { recursive: true, force: false });+ deps.removeFolderIndexEntries(operationPath);+ }+ deps.invalidateReferencedAssetsCache();++ purgeFileIndex(deletedDocNames);+ await deps.deleteDerivedDocumentsBestEffort(deletedDocNames, 'delete-path');+ deps.signalFiles();+ return { ok: true, deletedDocNames };+ },++ async trashCleanup(operationKind, path, operationDocName, logSource) {+ if (operationKind === 'asset') {+ deps.invalidateReferencedAssetsCache();+ deps.signalFiles();+ return { deletedDocNames: [] };+ }+ const initialIndex = deps.getFileIndex();+ const deletedDocNames =+ operationKind === 'file'+ ? initialIndex.has(operationDocName)+ ? [operationDocName]+ : []+ : deps.listAffectedDocNames(initialIndex, 'folder', path);++ deps.invalidateReferencedAssetsCache();++ if (deletedDocNames.length === 0) {+ return { deletedDocNames: [] };+ }++ await deps.captureAndCloseDocuments(deletedDocNames, 'deleted-upstream');+ markRemoved(deletedDocNames, logSource);+ purgeFileIndex(deletedDocNames);+ if (operationKind === 'folder') {+ deps.removeFolderIndexEntries(path);+ }+ await deps.deleteDerivedDocumentsBestEffort(deletedDocNames, 'trash-cleanup');+ // Synchronous CC1 emit closes the race where the renderer expects the+ // updated tree right after the response; the watcher's later emit is+ // idempotent at the consumer.+ deps.signalFiles();+ return { deletedDocNames };+ },++ async duplicatePath(kind, requestedPath, requestedDocName) {+ const sourcePath = deps.resolveContentEntryPath(contentDir, kind, requestedPath);+ if (!existsSync(sourcePath)) {+ if (kind === 'file') {+ const folderSourcePath = deps.resolveContentEntryPath(+ contentDir,+ 'folder',+ requestedPath,+ );+ if (existsSync(folderSourcePath) && statSync(folderSourcePath).isDirectory()) {+ return { ok: false, kind: 'type-mismatch' };+ }+ }+ return { ok: false, kind: 'not-found' };+ }+ const sourceStat = statSync(sourcePath);+ if (+ (kind === 'file' && !sourceStat.isFile()) ||+ (kind === 'folder' && !sourceStat.isDirectory())+ ) {+ return { ok: false, kind: 'type-mismatch' };+ }++ // Duplicating a conflicted source would copy the raw `<<<<<<< HEAD` /+ // `=======` / `>>>>>>>` marker bytes from disk into a new file at the+ // destination, producing a broken duplicate. Refuse; the user must+ // resolve the conflict first. Disk-truth enumeration for the same+ // watcher-lag reason as deletePath — a fresh, unindexed conflicted+ // child must not be silently skipped.+ const sourceDocNames =+ kind === 'file'+ ? [deps.docNameForPath(requestedPath)]+ : deps.listManagedDocNamesUnderFolder(+ deps.resolveContentEntryPath(contentDir, 'folder', requestedPath),+ );+ const conflictedFile = findConflictedFile(sourceDocNames);+ if (conflictedFile !== null) {+ return { ok: false, kind: 'conflict', file: conflictedFile };+ }++ if (kind === 'file') {+ const sourceExtension = extname(sourcePath);+ const { docName: duplicatedPath } = deps.nextAvailableDuplicateDocName(requestedDocName);+ if (+ isSystemDoc(duplicatedPath) ||+ isConfigDoc(duplicatedPath) ||+ deps.contentFilter?.isExcluded(`${duplicatedPath}${sourceExtension}`)+ ) {+ return { ok: false, kind: 'destination-excluded' };+ }+ const destinationPath = deps.resolveDuplicateDocPath(duplicatedPath, sourceExtension);+ const content = readFileSync(sourcePath, 'utf-8');+ const destinationDir = dirname(destinationPath);+ const destinationDirExisted = existsSync(destinationDir);+ try {+ tracedMkdirSync(destinationDir, { recursive: true });+ tracedWriteFileSync(destinationPath, content, { encoding: 'utf-8', flag: 'wx' });+ } catch (err) {+ if (isAlreadyExistsError(err)) {+ return { ok: false, kind: 'already-exists', cause: err };+ }+ if (!destinationDirExisted) {+ try {+ tracedRmdirSync(destinationDir);+ } catch (cleanupErr) {+ const cleanupCode = errnoCode(cleanupErr);+ if (cleanupCode !== 'ENOENT' && cleanupCode !== 'ENOTEMPTY') {+ log.warn(+ { destinationDir, err: cleanupErr },+ '[duplicate-path] failed to clean duplicate parent directory',+ );+ }+ }+ }+ throw err;+ }+ let didIncrementMdDir = false;+ try {+ registerDocExtension(duplicatedPath, sourceExtension);+ deps.unmarkRecentlyRemoved?.(duplicatedPath);+ if (deps.contentFilter) {+ deps.contentFilter.incrementMdDir(dirname(duplicatedPath));+ didIncrementMdDir = true;+ }+ registerWrite(destinationPath, contentHash(content));+ deps.mutateFileIndexCreate?.({+ path: destinationPath,+ docName: duplicatedPath,+ content,+ });+ } catch (err) {+ try {+ tracedRmSync(destinationPath, { force: true });+ } catch (cleanupErr) {+ log.warn(+ { destinationPath, err: cleanupErr },+ '[duplicate-path] failed to clean partial file duplicate',+ );+ }+ forgetDocExtension(duplicatedPath);+ if (deps.contentFilter && didIncrementMdDir) {+ deps.contentFilter.decrementMdDir(dirname(duplicatedPath));+ }+ deps.mutateFileIndexDelete?.({ path: destinationPath, docName: duplicatedPath });+ throw err;+ }+ await deps.recordDerivedDocumentBestEffort(duplicatedPath, content, 'duplicate-path-file');+ return { ok: true, duplicatedPath, duplicatedDocNames: [duplicatedPath] };+ }++ const { folderPath: duplicatedPath } = deps.nextAvailableDuplicateFolderPath(requestedPath);+ if (deps.contentFilter?.isDirExcluded(duplicatedPath)) {+ return { ok: false, kind: 'destination-excluded' };+ }+ const destinationPath = deps.resolveContentEntryPath(contentDir, 'folder', duplicatedPath);+ const copiedDocRollbackLedger: Array<{+ docName: string;+ fullPath: string;+ extensionRegistered: boolean;+ dirCountIncremented: boolean;+ fileIndexRegistered: boolean;+ }> = [];+ try {+ tracedCpSync(sourcePath, destinationPath, {+ recursive: true,+ errorOnExist: true,+ force: false,+ });+ } catch (err) {+ if (isAlreadyExistsError(err)) {+ return { ok: false, kind: 'already-exists', cause: err };+ }+ throw err;+ }+ const derivedMutations: DerivedDocumentIndexMutation[] = [];+ let duplicatedDocNames: string[] = [];+ try {+ for (const folderPath of deps.collectFolderPaths(duplicatedPath)) {+ deps.upsertFolderIndexPathSegments(folderPath);+ }+ const copiedDocs = deps.collectMarkdownCopies(duplicatedPath);+ duplicatedDocNames = copiedDocs.map((doc) => doc.docName);+ for (const doc of copiedDocs) {+ const rollbackEntry = {+ docName: doc.docName,+ fullPath: doc.fullPath,+ extensionRegistered: false,+ dirCountIncremented: false,+ fileIndexRegistered: false,+ };+ copiedDocRollbackLedger.push(rollbackEntry);+ const sourceExtension = extname(doc.fullPath);+ registerDocExtension(stripDocExtension(doc.docName), sourceExtension);+ rollbackEntry.extensionRegistered = true;+ deps.unmarkRecentlyRemoved?.(doc.docName);+ if (deps.contentFilter) {+ deps.contentFilter.incrementMdDir(dirname(doc.docName));+ rollbackEntry.dirCountIncremented = true;+ }+ registerWrite(doc.fullPath, contentHash(doc.content));+ rollbackEntry.fileIndexRegistered = true;+ deps.mutateFileIndexCreate?.({+ path: doc.fullPath,+ docName: doc.docName,+ content: doc.content,+ });+ derivedMutations.push({+ kind: 'upsert',+ documentName: doc.docName,+ markdown: doc.content,+ });+ }+ } catch (err) {+ for (const rollbackEntry of copiedDocRollbackLedger.reverse()) {+ if (rollbackEntry.fileIndexRegistered) {+ try {+ deps.mutateFileIndexDelete?.({+ path: rollbackEntry.fullPath,+ docName: rollbackEntry.docName,+ });+ } catch (rollbackErr) {+ log.warn(+ { docName: rollbackEntry.docName, err: rollbackErr },+ '[duplicate-path] failed to roll back copied file-index row',+ );+ }+ }+ if (rollbackEntry.dirCountIncremented) {+ deps.contentFilter?.decrementMdDir(dirname(rollbackEntry.docName));+ }+ if (rollbackEntry.extensionRegistered) {+ forgetDocExtension(stripDocExtension(rollbackEntry.docName));+ }+ }+ deps.removeFolderIndexEntries(duplicatedPath);+ try {+ tracedRmSync(destinationPath, { recursive: true, force: true });+ } catch (cleanupErr) {+ log.warn(+ { destinationPath, err: cleanupErr },+ '[duplicate-path] failed to clean partial folder duplicate',+ );+ }+ throw err;+ }+ await deps.recordDerivedMutationsBestEffort(derivedMutations, 'duplicate-path-folder');+ return { ok: true, duplicatedPath, duplicatedDocNames };+ },+ };+}diff --git a/packages/server/src/services/search.ts b/packages/server/src/services/search.tsnew file mode 100644index 000000000..608eea49c--- /dev/null+++ b/packages/server/src/services/search.ts@@ -0,0 +1,775 @@+import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from 'node:fs';+import { readFile } from 'node:fs/promises';+import { resolve } from 'node:path';+import {+ createWorkspaceSearchCorpus,+ createWorkspaceSearchDocument,+ isHiddenDocName,+ LEGACY_SKILL_STORE_ROOT,+ parseProjectSkillBundleDoc,+ type SearchSemanticStatus,+ type SearchSource,+ type SearchSuccess,+ SKILL_NAME_REGEX,+ searchWorkspaceCorpus,+ updateWorkspaceSearchCorpus,+ type WorkspaceSearchCorpus,+ type WorkspaceSearchDocument,+ type WorkspaceSearchIntent,+ type WorkspaceSearchRanking,+ type WorkspaceSearchResult,+ type WorkspaceSearchScope,+ type WorkspaceSemanticInput,+} from '@inkeep/open-knowledge-core';+import { isConfigDoc, isSystemDoc } from '../cc1-broadcast.ts';+import {+ recordSemanticQuery,+ type SemanticQueryOutcome,+} from '../embeddings/embeddings-telemetry.ts';+import { SEMANTIC_MIN_QUERY_LENGTH, type SemanticSearchService } from '../embeddings/index.ts';+import type { FileIndexEntry } from '../file-watcher.ts';+import { scanInPlaceSkills } from '../in-place-skills.ts';+import { getLogger } from '../logger.ts';+import { extractPageTitle } from '../page-identity.ts';+import { getMeter } from '../telemetry.ts';++/**+ * Workspace search: corpus build (page + skill + name-only file + folder+ * tiers), fingerprint-keyed caching with incremental index updates, the+ * cold-start warming gate, and the opt-in semantic blend. The transport+ * parses params and maps `SearchSuccess` onto the wire; everything between+ * lives here so GET and POST cannot drift in ranking, snippets, or the+ * semantic gate.+ */++const log = getLogger('search');++// Count of workspace search corpus builds, by mode: `cold` (first build),+// `incremental` (in-place index patch of only the changed documents), or+// `rebuild` (incremental path fell back to a from-scratch build; the bounded+// `reason` attribute says why). A steady stream of `rebuild` where+// `incremental` is expected is the drift signal worth alerting on.+let _searchCorpusUpdateCounter: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null =+ null;+function searchCorpusUpdateCounter(): ReturnType<ReturnType<typeof getMeter>['createCounter']> {+ _searchCorpusUpdateCounter ||= getMeter().createCounter('ok.search.corpus_update_total', {+ description:+ 'Count of workspace search corpus builds, by mode (cold | incremental | rebuild) and, for rebuilds, the bounded fallback reason.',+ });+ return _searchCorpusUpdateCounter;+}++let _searchCorpusTruncatedCounter: ReturnType<ReturnType<typeof getMeter>['createCounter']> | null =+ null;+function searchCorpusTruncatedCounter(): ReturnType<ReturnType<typeof getMeter>['createCounter']> {+ _searchCorpusTruncatedCounter ||= getMeter().createCounter('ok.search.corpus_truncated_total', {+ description:+ 'Count of corpus builds whose name-only file tier was truncated at OK_SEARCH_MAX_ENTRIES.',+ });+ return _searchCorpusTruncatedCounter;+}++interface WorkspaceSearchCacheEntry {+ fingerprint: string;+ corpus?: WorkspaceSearchCorpus;+ truncated?: boolean;+ pending?: Promise<{ corpus: WorkspaceSearchCorpus; truncated: boolean }>;+}++const workspaceSearchCaches = new Map<string, WorkspaceSearchCacheEntry>();++function deriveFolderSearchDocuments(+ pages: readonly WorkspaceSearchDocument[],+): WorkspaceSearchDocument[] {+ const folderModified = new Map<string, number>();+ for (const page of pages) {+ const segments = page.path.split('/').filter(Boolean);+ segments.pop();+ for (let i = 1; i <= segments.length; i++) {+ const folderPath = segments.slice(0, i).join('/');+ folderModified.set(+ folderPath,+ Math.max(folderModified.get(folderPath) ?? 0, page.modifiedTs),+ );+ }+ }+ return [...folderModified.entries()].map(([path, modifiedTs]) =>+ createWorkspaceSearchDocument({ kind: 'folder', path, modifiedTs }),+ );+}++function buildSearchSnippet(content: string, query: string): string | undefined {+ const normalizedQuery = query.trim().toLowerCase();+ if (!normalizedQuery || !content) return undefined;+ const normalizedContent = content.toLowerCase();+ const index = normalizedContent.indexOf(normalizedQuery);+ if (index < 0) return undefined;+ const start = Math.max(0, index - 80);+ const end = Math.min(content.length, index + normalizedQuery.length + 120);+ const prefix = start > 0 ? '…' : '';+ const suffix = end < content.length ? '…' : '';+ // slice() cuts on UTF-16 code units, so a boundary landing mid-emoji leaves a+ // lone surrogate. Replace any unpaired surrogate with U+FFFD so strict JSON-RPC+ // clients (Rust / pydantic parsers) don't reject the response as invalid UTF-8.+ // (String.toWellFormed() would do this but needs the es2024 lib in every consumer.)+ const snippet = `${prefix}${content.slice(start, end).replace(/\s+/g, ' ').trim()}${suffix}`;+ return snippet.replace(+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,+ '\uFFFD',+ );+}++/** Map a search result to the wire entry, carrying `vector` only when present. */+function toSearchResultEntry(+ result: ReturnType<typeof searchWorkspaceCorpus>[number],+ query: string,+): {+ kind: WorkspaceSearchScope;+ path: string;+ title: string;+ score: number;+ signals: WorkspaceSearchResult['signals'];+ snippet?: string;+} {+ return {+ kind: result.document.kind,+ path: result.document.path,+ title: result.document.title,+ score: result.score,+ signals: result.signals,+ snippet:+ result.document.kind === 'page'+ ? buildSearchSnippet(result.document.content, query)+ : undefined,+ };+}++// Per-entry change-detection key: the fields whose change should re-read a+// page (modified / size / canonical path / inode / aliases), NUL-separated so+// a path containing spaces can't merge fields and collide. `workspaceSearchFingerprint`'s+// fallback prefixes this with the docName; the page-doc cache keys on it+// directly (its Map is already docName-keyed). One definition keeps the two in+// lockstep — drift would silently break cache invalidation (stale reuse or+// needless re-reads).+function entrySearchKey(entry: FileIndexEntry): string {+ // NUL between fields AND between aliases: a path/alias containing a comma+ // (rare but valid on macOS/Linux) must not collide with a different alias set.+ return `${entry.modified}\0${entry.size}\0${entry.canonicalPath}\0${entry.inode}\0${entry.aliases.join('\0')}`;+}++interface SemanticResolution {+ /** Vector input for `searchWorkspaceCorpus`, or undefined for pure-lexical. */+ input?: WorkspaceSemanticInput;+ /** Non-content coverage status block to attach to the response. */+ status?: SearchSemanticStatus;+ /** Per-query embed latency (ms), or null when no query embed ran. */+ queryEmbedMs: number | null;+ /** Total embeddable pages (coverage denominator). */+ pageTotal: number;+ /** Whether the embedder is loaded + keyed + warm. */+ capable: boolean;+}++export interface SearchServiceDeps {+ contentDir: string;+ projectDir?: string;+ getAllFilesIndex: () => ReadonlyMap<string, FileIndexEntry>;+ getFileIndexGeneration?: () => number;+ getSearchMaxEntries: () => number;+ semanticSearch?: SemanticSearchService;+ getSemanticSimilarityFloor?: () => number | undefined;+ /** Boot index seed gate; absent (test harnesses) means ready immediately. */+ ready?: Promise<unknown>;+ /** The project-scope legacy skill-store root. */+ getProjectSkillsRoot: () => string;+ parseFrontmatterDoc: (raw: string) => { frontmatter: Record<string, unknown>; body: string };+}++export interface SearchService {+ buildSearchResponse(params: {+ query: string;+ intent: WorkspaceSearchIntent;+ ranking: WorkspaceSearchRanking | undefined;+ scopes: WorkspaceSearchScope[] | undefined;+ limit: number | undefined;+ semanticParam: boolean | undefined;+ source: SearchSource;+ }): Promise<SearchSuccess>;+ prewarm(): void;+}++export function createSearchService(deps: SearchServiceDeps): SearchService {+ const {+ contentDir,+ projectDir,+ getAllFilesIndex,+ getFileIndexGeneration,+ semanticSearch,+ getSemanticSimilarityFloor,+ ready,+ } = deps;++ /**+ * Resolve the per-query vector signal + coverage status for a search.+ *+ * Returns a pure-lexical resolution (no `input`, no `status`) — byte-identical+ * to the pre-embeddings path — unless the feature flag is ON **and** the caller+ * opted in (`semantic: true`). The omnibar and `semantic: false` never opt in,+ * so they stay lexical and carry no status block. When opted-in, fires the lazy+ * background corpus embed (no-op when incapable) and embeds only the query.+ */+ async function resolveSemantic(+ query: string,+ intent: WorkspaceSearchIntent,+ semanticParam: boolean | undefined,+ corpus: WorkspaceSearchCorpus,+ ): Promise<SemanticResolution> {+ // Predicate split: hidden / dot-path docs are searchable (admitted to the+ // corpus) but NEVER embedded — no semantic egress for agent-tooling/dotfiles.+ // The embeddable set is the corpus minus hidden paths, and it also drives the+ // coverage denominator so a searchable dot-path page is never counted as+ // "embeddable" (which would make coverage under-report forever).+ const embeddableDocs = corpus.documents.filter((d) => !isHiddenDocName(d.path));+ const pageTotal = embeddableDocs.reduce((n, d) => n + (d.kind === 'page' ? 1 : 0), 0);+ // Flag OFF, or the caller did not opt in → no status block, lexical path.+ if (!semanticSearch?.isEnabled() || semanticParam !== true) {+ return { queryEmbedMs: null, pageTotal, capable: false };+ }++ // Opted in + enabled: lazily (re-)embed the corpus in the background. Cheap+ // for unchanged docs; no-op when no key. This is the only embed trigger —+ // nothing embeds until an agent actually searches (no proactive egress).+ void semanticSearch.embedCorpus(embeddableDocs);++ // Semantic fuses into the body blend only, and skips trivially short queries.+ let input: WorkspaceSemanticInput | undefined;+ let queryEmbedMs: number | null = null;+ if (intent === 'full_text' && query.trim().length >= SEMANTIC_MIN_QUERY_LENGTH) {+ const startedAt = performance.now();+ const scores = await semanticSearch.queryScores(query, embeddableDocs);+ queryEmbedMs = performance.now() - startedAt;+ if (scores && scores.size > 0) {+ // Carry the project-local similarity floor when set so a model whose+ // cosine scale differs from the default can be retuned without a code+ // change; undefined leaves core on its model-calibrated default.+ const similarityFloor = getSemanticSimilarityFloor?.();+ input = similarityFloor !== undefined ? { scores, similarityFloor } : { scores };+ }+ }++ const status = semanticSearch.getStatus();+ return {+ input,+ status: {+ capable: status.capable,+ applied: false, // finalized post-ranking (did any result carry a vector)+ coverage: { embedded: status.embeddedCount, total: pageTotal },+ },+ queryEmbedMs,+ pageTotal,+ capable: status.capable,+ };+ }++ /**+ * Project skills (`<root>/.ok/skills/<name>/SKILL.md`) as cheap stat records —+ * readdir + stat only, no content read — so the per-search corpus fingerprint+ * can detect skill changes without paying a content read on every request.+ * Skills are tree-excluded from `getFileIndex()`, so search enumerates them+ * from disk. The corpus doc builder reuses this list and reads each matched+ * file's content.+ */+ function enumerateProjectSkillStats(): Array<{+ name: string;+ absolutePath: string;+ /** The skill's LIVE content-doc path (`<real dir>/SKILL`) — what search+ * hits open. In-place skills carry their editor-dir path; only a legacy+ * store resident still carries the `.ok/skills` shape. */+ docPath: string;+ mtimeMs: number;+ size: number;+ }> {+ const out: Array<{+ name: string;+ absolutePath: string;+ docPath: string;+ mtimeMs: number;+ size: number;+ }> = [];+ const seen = new Set<string>();+ // In-place skills FIRST (they win a name collision with a store resident,+ // mirroring the list/read rules). Enumerating only the store root here was+ // a store-retirement fossil: in-place skills vanished from search entirely,+ // and the ones indexed opened phantom `.ok/skills` tabs.+ for (const skill of scanInPlaceSkills(contentDir)) {+ const skillMd = resolve(contentDir, skill.dir, 'SKILL.md');+ try {+ const st = statSync(skillMd);+ out.push({+ name: skill.name,+ absolutePath: skillMd,+ docPath: `${skill.dir}/SKILL`,+ mtimeMs: st.mtimeMs,+ size: st.size,+ });+ seen.add(skill.name);+ } catch {+ // Vanished between scan and stat — skip.+ }+ }+ const root = deps.getProjectSkillsRoot();+ if (!existsSync(root)) return out;+ let entries: Dirent[];+ try {+ entries = readdirSync(root, { withFileTypes: true });+ } catch {+ return out;+ }+ for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {+ if (!entry.isDirectory() || !SKILL_NAME_REGEX.test(entry.name)) continue;+ if (seen.has(entry.name)) continue;+ const skillMd = resolve(root, entry.name, 'SKILL.md');+ try {+ const st = statSync(skillMd);+ out.push({+ name: entry.name,+ absolutePath: skillMd,+ docPath: `${LEGACY_SKILL_STORE_ROOT}/${entry.name}/SKILL`,+ mtimeMs: st.mtimeMs,+ size: st.size,+ });+ } catch {+ // Missing/unreadable SKILL.md — skip (a draft dir with no manifest).+ }+ }+ return out;+ }++ /**+ * Project skills as search documents (keyword + semantic — `embedCorpus`+ * embeds every corpus doc). Indexed under their managed-artifact doc path so a+ * hit opens the skill tab via the shared nav resolution. Title is the skill's+ * frontmatter name; content is its description + body, so a skill is findable+ * by what it does, not just its slug.+ */+ function buildSkillSearchDocuments(): WorkspaceSearchDocument[] {+ const docs: WorkspaceSearchDocument[] = [];+ for (const skill of enumerateProjectSkillStats()) {+ let title = skill.name;+ let content = '';+ try {+ const { frontmatter, body } = deps.parseFrontmatterDoc(+ readFileSync(skill.absolutePath, 'utf-8'),+ );+ if (typeof frontmatter.name === 'string' && frontmatter.name) title = frontmatter.name;+ const desc = typeof frontmatter.description === 'string' ? frontmatter.description : '';+ content = `${desc}\n\n${body}`.trim();+ } catch {+ // Malformed/unreadable — index by name only so it is still findable.+ }+ docs.push(+ createWorkspaceSearchDocument({+ kind: 'page',+ // Index under the skill's LIVE content-doc path (real dir), never a+ // minted shape — a minted `__skill__/project/<name>` OR `.ok/skills`+ // path both made a search hit open a blank phantom tab.+ path: skill.docPath,+ title,+ content,+ modifiedTs: skill.mtimeMs,+ }),+ );+ }+ return docs;+ }++ // Per-page parsed-document cache. Building the corpus re-reads every markdown+ // file from disk, but a rebuild is triggered by ANY file-index change (one+ // edit, a rename, a new sibling), so without this every keystroke-after-an-edit+ // would re-read and re-parse the whole workspace. Reuse a page's search+ // document across rebuilds when its own entry is unchanged — re-reading only+ // the delta. Invariant direction: a change that busts THIS page's `entrySearchKey`+ // also bumps the generation counter that invalidates the corpus — but NOT the+ // converse: a rebuild triggered by a sibling change reuses this page's cached+ // doc when its own entry is unchanged (the whole point). Only successful reads+ // are cached, so a transient read failure self-heals on the next rebuild rather+ // than pinning empty content. Pruned to the live index each build, so it stays+ // bounded by the workspace size. The name-only `file` tier and derived folder+ // docs are metadata-only (no disk read), so they are rebuilt each time.+ const pageDocCache = new Map<string, { key: string; doc: WorkspaceSearchDocument }>();++ async function buildWorkspaceSearchDocumentsFromIndex(): Promise<{+ documents: WorkspaceSearchDocument[];+ truncated: boolean;+ }> {+ const pages: WorkspaceSearchDocument[] = [];+ const files: WorkspaceSearchDocument[] = [];+ const seenPages: Set<string> = new Set();+ for (const [docName, entry] of getAllFilesIndex()) {+ // System + config synthetic docs never enter search. Hidden / dot-prefixed+ // paths (`.changeset/`, `.github/`, `.cursor/`) DO — they are searchable by+ // name/path (rank-deprioritized in core) so "search what the tree shows"+ // holds. They stay out of the embedding/egress path, which keeps the+ // `isHiddenDocName` filter where the corpus is handed to the embedder.+ if (isSystemDoc(docName) || isConfigDoc(docName)) continue;+ // Project-skill SKILL docs ARE in the index (skills-as-content), and this+ // loop iterates the all-files view — but `buildSkillSearchDocuments()`+ // already indexes each skill with skill-aware title/content under the same+ // path. Skip ANY project SKILL-doc shape (in-place editor dirs AND the+ // legacy store) so one isn't added twice (a duplicate corpus id throws and+ // 500s the whole search). Bundle reference docs stay — they index as+ // ordinary pages.+ if (parseProjectSkillBundleDoc(docName)?.kind === 'skill') continue;+ if (docName.startsWith('.ok/skills/')) continue;+ if (entry.kind === 'file') {+ // Name-only tier: a non-markdown file is searchable by name / path /+ // folder, but its body is NEVER read (content stays markdown-only).+ // `pathToDocName` keeps the extension for non-markdown, so `data.csv`+ // is findable by both `data` and `data.csv`; the basename is the title.+ files.push(+ createWorkspaceSearchDocument({+ kind: 'file',+ path: docName,+ modifiedTs: Date.parse(entry.modified),+ // Symlink alias paths fold into searchable pathSegments (inode-dedup+ // already gives one entry per file via the canonical-keyed index).+ aliases: entry.aliases,+ }),+ );+ continue;+ }+ // Markdown page: reuse the cached parse when its entry is unchanged (same+ // fingerprint components), else re-read and re-cache.+ seenPages.add(docName);+ const entryKey = entrySearchKey(entry);+ const cached = pageDocCache.get(docName);+ if (cached && cached.key === entryKey) {+ pages.push(cached.doc);+ continue;+ }+ let content = '';+ let title = docName;+ let readFailed = false;+ try {+ content = await readFile(entry.canonicalPath, 'utf-8');+ } catch (err) {+ // A transient read (external editor mid-save, EBUSY, NFS blip, a+ // watcher-vs-disk race) must NOT be cached — the entry fingerprint does+ // not change just because the read failed, so a cached empty-content doc+ // would persist and silently hide the page from body search until its+ // mtime/size/inode shifts. Skip the cache write so the next rebuild+ // retries, preserving the pre-cache self-healing behavior.+ readFailed = true;+ log.warn({ docName, err }, `[search] Failed to read ${docName}`);+ }+ if (!readFailed) {+ try {+ title = extractPageTitle(content, docName);+ } catch (err) {+ // Title extraction is pure string work, so a throw here is a+ // deterministic parse fault, not transient I/O. Fall back to the+ // docName as title but still cache (the read succeeded) — caching it+ // avoids re-parsing the same failing content on every rebuild, the+ // opposite of the read-failure path's deliberate retry.+ log.warn({ docName, err }, `[search] Failed to extract title for ${docName}`);+ }+ }+ const doc = createWorkspaceSearchDocument({+ kind: 'page',+ path: docName,+ title,+ content,+ modifiedTs: Date.parse(entry.modified),+ aliases: entry.aliases,+ });+ if (!readFailed) pageDocCache.set(docName, { key: entryKey, doc });+ pages.push(doc);+ }+ // Prune cache entries for pages no longer in the index (deleted / renamed)+ // so the cache tracks the live workspace rather than growing unbounded.+ // Unconditional: a failed read adds to `seenPages` but not to the cache, so+ // a `size`-comparison guard could read equal and skip a genuinely-needed+ // prune. The loop is O(cache) — same order as the build it follows.+ for (const docName of pageDocCache.keys()) {+ if (!seenPages.has(docName)) pageDocCache.delete(docName);+ }+ // Cap the name-only file tier (markdown pages are never dropped). Over the+ // ceiling, drop DEEPEST paths first (level-order): the shallowest entries are+ // the most navigationally useful, and dropping the deep tail mirrors the+ // show-all truncation BFS. The dogfood repo (~16k) is far under the+ // 50k default; this is a pathological-repo backstop.+ const maxFiles = deps.getSearchMaxEntries();+ let admittedFiles = files;+ let truncated = false;+ if (files.length > maxFiles) {+ truncated = true;+ admittedFiles = [...files]+ .sort((a, b) => {+ const depthA = a.path.split('/').length;+ const depthB = b.path.split('/').length;+ return depthA - depthB || a.path.localeCompare(b.path);+ })+ .slice(0, maxFiles);+ // Surface the cap-fire to operators: a structured warn log + a meter+ // counter. Without these the cap is silent — operators see "search+ // missing some files" with no signal pointing at `OK_SEARCH_MAX_ENTRIES`.+ // One emission per corpus rebuild (the cache then absorbs subsequent+ // queries until the fingerprint changes).+ log.warn(+ {+ dropped: files.length - admittedFiles.length,+ retained: admittedFiles.length,+ limit: maxFiles,+ },+ '[search] corpus name-only file tier truncated at OK_SEARCH_MAX_ENTRIES',+ );+ searchCorpusTruncatedCounter().add(1);+ }+ // Folders are synthesized from ALL admitted paths (markdown pages + name-only+ // file entries), so a folder containing only non-markdown files is still a+ // search result and a partial-path query (e.g. `server/src`) resolves even+ // when the folder holds no markdown.+ const documents = [+ ...pages,+ ...buildSkillSearchDocuments(),+ ...admittedFiles,+ ...deriveFolderSearchDocuments([...pages, ...admittedFiles]),+ ];+ return { documents, truncated };+ }++ // Stat-only skill fingerprint (name + mtime + size per project skill). A+ // named helper, not a local `const`, so the getAllFilesIndex caller-coverage+ // meta-test attributes the call in `workspaceSearchFingerprint` to that+ // allowlisted function rather than to an intermediate binding.+ function skillStatFingerprint(): string {+ return enumerateProjectSkillStats()+ .map((s) => `${s.name} ${s.mtimeMs} ${s.size}`)+ .join('');+ }++ function workspaceSearchFingerprint(): string {+ // Skills are tree-excluded from the file index, so neither the generation+ // counter nor getAllFilesIndex reflects a skill add/edit/remove. Fold the+ // stat-only skill fingerprint into BOTH paths so the corpus rebuilds on a+ // skill change (no content read on the per-search fingerprint path).+ // Fast path: the watcher's monotonic generation counter bumps on every+ // file-index mutation (the same counter that memoizes the markdown-only+ // view), so a generation match proves the corpus is still valid in O(1).+ if (getFileIndexGeneration) {+ return `gen:${getFileIndexGeneration()}|skills${skillStatFingerprint()}`;+ }+ // Fallback for harnesses that wire only the index accessors. Admission+ // predicate MUST match `buildWorkspaceSearchDocumentsFromIndex` so a+ // change to a now-searchable dot-path busts the corpus cache.+ return `${[...getAllFilesIndex()]+ .filter(([docName]) => !isSystemDoc(docName) && !isConfigDoc(docName))+ .sort(([a], [b]) => a.localeCompare(b))+ .map(+ // Shares `entrySearchKey` with the page-doc cache so the two never drift.+ ([docName, entry]) => `${docName}\0${entrySearchKey(entry)}`,+ )+ .join('')}|skills${skillStatFingerprint()}`;+ }++ // Cold-start search readiness. While the boot index seed is still walking the+ // content dir, `/api/search` must not block on it nor return a false-empty+ // result: an agent (MCP `search`) or any consumer hitting search right after+ // `ok start` would otherwise get zero hits that read as complete. We answer+ // fast with `ready: false` instead and let the caller retry. The command+ // palette gates its own fetch on the page-list cold-load signal, so this+ // primarily protects non-UI consumers and is defense-in-depth for the UI.+ //+ // `bootIndexReady` mirrors the same boot gate `handleDocumentList` awaits: an+ // absent gate (test harnesses) is ready immediately, and a rejected gate still+ // flips ready (logged, like the sibling document-list gate) so a degraded boot+ // serves whatever index exists rather than warming forever.+ let bootIndexReady = ready === undefined;+ ready?.then(+ () => {+ bootIndexReady = true;+ },+ (err: unknown) => {+ bootIndexReady = true;+ log.warn(+ { err, handler: 'search' },+ '[api] ready gate rejected — search serves the partial index',+ );+ },+ );++ // Warming = the boot seed has not finished. Once it has, search awaits the+ // corpus build and returns results as before (the lazy first build is fast and+ // prewarmed; a slow first build on a very large workspace is the documented+ // residual). Scoping warming to the seed window keeps steady-state behavior —+ // and every consumer that does not pass a boot gate — unchanged.+ function isSearchCorpusWarming(): boolean {+ return !bootIndexReady;+ }++ async function getWorkspaceSearchCorpus(): Promise<{+ corpus: WorkspaceSearchCorpus;+ truncated: boolean;+ }> {+ const cacheKey = `${contentDir} ${projectDir ?? ''}`;+ const fingerprint = workspaceSearchFingerprint();+ const workspaceSearchCache = workspaceSearchCaches.get(cacheKey);+ if (workspaceSearchCache?.fingerprint === fingerprint && workspaceSearchCache.corpus) {+ return {+ corpus: workspaceSearchCache.corpus,+ truncated: workspaceSearchCache.truncated ?? false,+ };+ }+ if (workspaceSearchCache?.fingerprint === fingerprint && workspaceSearchCache.pending) {+ return workspaceSearchCache.pending;+ }++ // Stale-but-live corpus (or the build about to produce one): the base for+ // an incremental index patch, so one write re-indexes one document instead+ // of re-tokenizing the whole workspace on the event loop.+ const priorCorpus = workspaceSearchCache?.corpus;+ const priorPending = workspaceSearchCache?.pending;+ const pending = (async () => {+ // Serialize behind any in-flight build: an incremental diff is only valid+ // against the corpus it was computed from, and the in-flight build owns+ // the shared index right now. Chaining keeps updates linear (each build+ // bases on its predecessor's output) without coalescing away freshness —+ // the document snapshot below is read AFTER this fingerprint was seen.+ const base = priorPending+ ? await priorPending.then(+ (result) => result.corpus,+ () => undefined,+ )+ : priorCorpus;+ const { documents, truncated } = await buildWorkspaceSearchDocumentsFromIndex();+ if (!base) {+ searchCorpusUpdateCounter().add(1, { mode: 'cold' });+ return { corpus: createWorkspaceSearchCorpus(documents), truncated };+ }+ const update = updateWorkspaceSearchCorpus(base, documents);+ if (update.rebuilt) {+ searchCorpusUpdateCounter().add(1, { mode: 'rebuild', reason: update.rebuildReason });+ // `mutation-failed` means the patched index diverged from the document+ // set (or a mutation threw) — recovered by the rebuild, but worth an+ // operator-visible signal; the elective reasons are routine.+ const logLevel = update.rebuildReason === 'mutation-failed' ? 'warn' : 'debug';+ log[logLevel](+ { reason: update.rebuildReason, documents: documents.length },+ '[search] corpus update fell back to a full index rebuild',+ );+ } else {+ searchCorpusUpdateCounter().add(1, { mode: 'incremental' });+ log.debug(+ { inserted: update.inserted, updated: update.updated, removed: update.removed },+ '[search] corpus updated incrementally',+ );+ }+ return { corpus: update.corpus, truncated };+ })();+ workspaceSearchCaches.set(cacheKey, { fingerprint, pending });+ try {+ const result = await pending;+ if (workspaceSearchCaches.get(cacheKey)?.pending === pending) {+ workspaceSearchCaches.set(cacheKey, {+ fingerprint,+ corpus: result.corpus,+ truncated: result.truncated,+ });+ }+ return result;+ } catch (err) {+ if (workspaceSearchCaches.get(cacheKey)?.pending === pending) {+ workspaceSearchCaches.delete(cacheKey);+ }+ throw err;+ }+ }++ return {+ /**+ * Shared core for `GET` + `POST /api/search`: build the corpus, resolve the+ * (opt-in) vector signal, rank, and assemble the `SearchSuccess` body. One+ * implementation so GET and POST cannot drift in ranking, snippets, or the+ * semantic gate.+ */+ async buildSearchResponse(params) {+ const startedAt = performance.now();+ // Cold start: while the boot seed is still walking the content dir, do not+ // block on it and do not serve a partial/empty index as if it were complete.+ // Answer fast with `ready: false` so the caller (MCP `search`, palette, any+ // consumer) retries instead of trusting an empty result. The seed populates+ // the file index, so a retry after it resolves takes the normal path below.+ if (isSearchCorpusWarming()) {+ return {+ query: params.query,+ intent: params.intent,+ results: [],+ elapsedMs: Math.max(0, performance.now() - startedAt),+ ready: false,+ };+ }+ const { corpus, truncated } = await getWorkspaceSearchCorpus();+ const semantic = await resolveSemantic(+ params.query,+ params.intent,+ params.semanticParam,+ corpus,+ );+ const results = searchWorkspaceCorpus(corpus, params.query, {+ intent: params.intent,+ ranking: params.ranking,+ scopes: params.scopes,+ limit: params.limit,+ semantic: semantic.input,+ });+ const entries = results.map((r) => toSearchResultEntry(r, params.query));++ let semanticStatus: SearchSemanticStatus | undefined;+ if (semantic.status) {+ const vectorContributors = entries.reduce(+ (n, e) => n + (e.signals.vector !== undefined ? 1 : 0),+ 0,+ );+ const applied = vectorContributors > 0;+ semanticStatus = { ...semantic.status, applied };+ const outcome: SemanticQueryOutcome = !semantic.capable+ ? 'incapable'+ : applied+ ? 'applied'+ : semantic.status.coverage.embedded === 0+ ? 'warming'+ : 'no_match';+ recordSemanticQuery({+ outcome,+ source: params.source,+ capable: semantic.capable,+ embedded: semantic.status.coverage.embedded,+ total: semantic.pageTotal,+ queryEmbedMs: semantic.queryEmbedMs,+ vectorContributors,+ });+ }++ return {+ query: params.query,+ intent: params.intent,+ results: entries,+ elapsedMs: Math.max(0, performance.now() - startedAt),+ ready: true,+ ...(semanticStatus ? { semantic: semanticStatus } : {}),+ ...(truncated ? { truncated: true } : {}),+ };+ },++ prewarm() {+ if (process.env.NODE_ENV === 'test') return;+ for (const delayMs of [0, 1000, 3000]) {+ setTimeout(() => {+ void getWorkspaceSearchCorpus().catch((err) => {+ log.warn({ err }, '[search] Failed to prewarm workspace search cache');+ });+ }, delayMs);+ }+ },+ };+}diff --git a/packages/server/src/services/skill-import.ts b/packages/server/src/services/skill-import.tsnew file mode 100644index 000000000..e266eebe7--- /dev/null+++ b/packages/server/src/services/skill-import.ts@@ -0,0 +1,378 @@+import { existsSync, lstatSync } from 'node:fs';+import { join, resolve } from 'node:path';+import {+ SKILL_IMPORT_MAX_BUNDLE_FILES,+ SKILL_IMPORT_MAX_FILE_BYTES,+ SKILL_IMPORT_MAX_TOTAL_BYTES,+ type SkillImportSuccessSchema,+} from '@inkeep/open-knowledge-core';+import {+ acquiredBundleTooLarge,+ findByContentHash,+ inspectPluginSource,+ parseSkillDir,+ SKILLS_LOCK_REL,+ upsertLockEntry,+} from '@inkeep/open-knowledge-core/skills-catalog';+import type { z } from 'zod';+import {+ applySkillBundleFileWrite,+ applySkillDelete,+ applySkillWrite,+} from '../content/skills-write.ts';+import type { extractActorIdentity } from '../extract-actor-identity.ts';+import { sanitizeFilename } from '../filename-sanitize.ts';+import { resolveDefaultSkillHomeRel } from '../in-place-skills.ts';+import { getLogger } from '../logger.ts';+import {+ projectInPlaceSkill,+ resolveSkillTargets,+ skillHostDir,+ skillProjectionRoots,+} from '../skill-projection.ts';+import { mutateSkillsLock, readSkillsLockFile } from '../skills-lock-store.ts';++/**+ * The shared skill-import spine (steps 2-6): given an on-disk skill dir ++ * provenance, dedupe by content hash, land under a free `-imported` name on+ * collision, write SKILL.md + bundle files via the sanctioned writers,+ * attribute + shadow-commit (project scope), record the lockfile entry, and+ * best-effort project into editor dirs. Returns an outcome; the transport+ * writes the response. Used by `/api/skill/import` (by-reference),+ * `/api/skill-upload` (by-bytes), and `/api/skills/import-bulk` (many+ * skills, one clone). The CALLER owns temp-dir cleanup (its `finally`), so+ * this never cleans up.+ */++const log = getLogger('skill-import');++type ActorIdentity = ReturnType<typeof extractActorIdentity>;++export const SKILL_IMPORT_WRITE_LIMITS = {+ maxFileBytes: SKILL_IMPORT_MAX_FILE_BYTES,+ maxFiles: SKILL_IMPORT_MAX_BUNDLE_FILES,+} as const;++export function importedBundleLimitError(+ acquired: NonNullable<ReturnType<typeof parseSkillDir>>,+): string | null {+ if (acquired.files.length > SKILL_IMPORT_MAX_BUNDLE_FILES) {+ return `Skill has ${acquired.files.length} dependent files; the import cap is ${SKILL_IMPORT_MAX_BUNDLE_FILES}.`;+ }+ let totalBytes = Buffer.byteLength(acquired.skillMd);+ for (const file of acquired.files) {+ const bytes = file.bytes?.byteLength ?? Buffer.byteLength(file.content ?? '');+ if (bytes > SKILL_IMPORT_MAX_FILE_BYTES) {+ return `${file.relPath} is ${bytes} bytes; the import per-file cap is ${SKILL_IMPORT_MAX_FILE_BYTES}.`;+ }+ totalBytes += bytes;+ }+ return totalBytes > SKILL_IMPORT_MAX_TOTAL_BYTES+ ? `Skill is ${totalBytes} bytes; the import bundle cap is ${SKILL_IMPORT_MAX_TOTAL_BYTES}.`+ : null;+}++/**+ * What the spine concluded. Returned rather than written so the bulk path+ * can run it once per skill against a SINGLE clone and report per-skill+ * rows; the transport's `respondSkillImport` applies the single-skill HTTP+ * shape.+ */+export type SkillImportOutcome =+ | { ok: true; body: z.infer<typeof SkillImportSuccessSchema> }+ | {+ ok: false;+ status: 400 | 422 | 500;+ urn: 'urn:ok:error:invalid-request' | 'urn:ok:error:internal-server-error';+ title: string;+ detail?: string;+ cause?: unknown;+ };++export interface SkillImportDeps {+ contentDir: string;+ skillsHome: string;+ projectDir?: string;+ resolveSkillDirForRead: (scope: 'project' | 'global', name: string) => string | null;+ parseFrontmatterDoc: (raw: string) => { frontmatter: Record<string, unknown>; body: string };+ attributeOkArtifactWrite: (actor: ActorIdentity, keyPath: string, summary: string) => void;+ commitOkArtifactWrite: (context: string) => Promise<void>;+ shadowHeadSha: (writerId?: string, verifyPathRel?: string) => Promise<string | undefined>;+ artifactWriterId: (actor: ActorIdentity) => string | undefined;+ effectiveInstallMode: (+ scope: 'project' | 'global',+ name: string,+ entry: { hosts: readonly string[]; linkedHosts: readonly string[] },+ ) => 'copy' | 'link';+ signalFiles: () => void;+}++export interface SkillImportService {+ runSkillImport(params: {+ acquiredDir: string;+ scope: 'project' | 'global';+ sourceLabel: string;+ ref?: string;+ publisher?: string;+ upstreamSkill?: string;+ actor: ActorIdentity;+ /** Skip the default-editor auto-projection — for callers that install+ * explicitly afterwards (explore preview toggles / custom-path place). */+ skipProjection?: boolean;+ }): Promise<SkillImportOutcome>;+}++export function createSkillImportService(deps: SkillImportDeps): SkillImportService {+ function localSkillHash(skillsRoot: string, name: string): string | undefined {+ return parseSkillDir(resolve(skillsRoot, name))?.contentHash;+ }++ return {+ async runSkillImport(params) {+ const { acquiredDir, scope, sourceLabel, ref, publisher, upstreamSkill, actor } = params;+ if (!deps.projectDir) {+ return {+ ok: false,+ status: 400,+ urn: 'urn:ok:error:invalid-request',+ title: 'No project root resolved.',+ detail: 'NO_PROJECT_ROOT',+ };+ }+ // Pre-flight the caps by stat, BEFORE the parse materializes every byte —+ // `importedBundleLimitError` below runs on an already-built array, which a+ // repo of half-gigabyte blobs never reaches.+ const oversize = acquiredBundleTooLarge(acquiredDir);+ if (oversize) {+ return {+ ok: false,+ status: 400,+ urn: 'urn:ok:error:invalid-request',+ title: 'Skill bundle exceeds import limits.',+ detail: oversize,+ };+ }+ const acquired = parseSkillDir(acquiredDir);+ if (!acquired) {+ return {+ ok: false,+ status: 422,+ urn: 'urn:ok:error:invalid-request',+ title: 'Source has no readable skill.',+ };+ }+ const limitError = importedBundleLimitError(acquired);+ if (limitError) {+ return {+ ok: false,+ status: 400,+ urn: 'urn:ok:error:invalid-request',+ title: 'Skill bundle exceeds import limits.',+ detail: limitError,+ };+ }++ // 2. Dedupe by contentHash via the lockfile (re-import is a no-op).+ const lockPath = join(+ scope === 'project' ? deps.projectDir : deps.skillsHome,+ ...SKILLS_LOCK_REL,+ );+ const lock = readSkillsLockFile(lockPath);+ // WRITE-PATH INVERSION (store retirement, step 1): NEW imports land+ // IN-PLACE at the vendor-neutral `.agents/skills` hub — versioned, listed,+ // and read at their real path from day one. The store gains no new+ // residents; existing store bundles keep working until their migration.+ const importBase = scope === 'project' ? deps.contentDir : deps.skillsHome;+ const importHomeRel = resolveDefaultSkillHomeRel(importBase, scope);+ const importRoot = resolve(importBase, importHomeRel);+ const dupName = findByContentHash(lock, acquired.contentHash);+ if (dupName && deps.resolveSkillDirForRead(scope, dupName) !== null) {+ return {+ ok: true,+ body: {+ name: dupName,+ path: `${dupName}/SKILL.md`,+ created: false,+ alreadyImported: true,+ provenance: { source: sourceLabel, ref, contentHash: acquired.contentHash, publisher },+ warnings: [],+ },+ };+ }++ // 3. Collision with an existing skill → land under a free `-imported` name.+ // `acquired.name` is attacker-controlled (imported SKILL.md frontmatter).+ // applySkillWrite's validateName blocks a malformed WRITE, but the existence+ // probes below resolve the name against skillsRoot — sanitize first so a+ // `../…` name can't turn firstFreeSkillName into a filesystem existence+ // oracle. No-op for a normal `[a-z0-9-]` name. (defense-in-depth)+ const probeName = sanitizeFilename(acquired.name);+ // Collision probe spans the WHOLE registry (store + every in-place host+ // dir), not just the import root — a same-name skill at `.claude/skills`+ // would fork against the new `.agents` bundle.+ const nameTaken = (n: string) =>+ deps.resolveSkillDirForRead(scope, n) !== null ||+ existsSync(resolve(importRoot, n, 'SKILL.md'));+ const collided = nameTaken(probeName);+ let targetName = probeName;+ if (collided) {+ targetName = `${probeName}-imported`;+ let n = 2;+ while (nameTaken(targetName)) {+ targetName = `${probeName}-imported-${n}`;+ n += 1;+ }+ }++ // 4. Write SKILL.md + every bundle file via the sanctioned writers. The+ // frontmatter is canonicalized to {name,description} (OK's skill model);+ // upstream version lives in the lockfile, not the SKILL.md.+ const skillBody = deps.parseFrontmatterDoc(acquired.skillMd).body;+ const wr = applySkillWrite({+ skillsRoot: importRoot,+ name: targetName,+ body: skillBody,+ frontmatter: { name: targetName, description: acquired.description },+ });+ if (!wr.ok) {+ return {+ ok: false,+ status: 400,+ urn: 'urn:ok:error:invalid-request',+ title: 'Failed to write imported skill.',+ detail: wr.error.code,+ cause: new Error(wr.error.message),+ };+ }+ const warnings = [...wr.warnings];+ for (const f of acquired.files) {+ const br = applySkillBundleFileWrite({+ skillsRoot: importRoot,+ name: targetName,+ relPath: f.relPath,+ content: f.content,+ bytes: f.bytes,+ limits: SKILL_IMPORT_WRITE_LIMITS,+ });+ if (!br.ok) {+ const rollback = applySkillDelete({ skillsRoot: importRoot, name: targetName });+ return {+ ok: false,+ status: rollback.ok ? 400 : 500,+ urn: rollback.ok+ ? 'urn:ok:error:invalid-request'+ : 'urn:ok:error:internal-server-error',+ title: 'Failed to write the complete imported skill bundle.',+ detail: `${f.relPath}: ${br.error.code}`,+ cause: new Error(+ rollback.ok+ ? br.error.message+ : `${br.error.message}; rollback: ${rollback.error.message}`,+ ),+ };+ }+ }++ // 5. Attribute + shadow-commit (project scope; global is unversioned).+ if (scope === 'project') {+ // Attribute under the content-doc key (`<dir>/SKILL`) that `/api/history`+ // filters on — same fix as the create path; the bare dir key fell through+ // the OkActor match so an imported skill's first version never showed in+ // (or was restorable from) its history.+ deps.attributeOkArtifactWrite(+ actor,+ `${importHomeRel}/${targetName}/SKILL`,+ `skill-import: ${sourceLabel} -> ${importHomeRel}/${targetName}/SKILL.md`,+ );+ await deps.commitOkArtifactWrite('skill-import');+ }++ // 6. Record upstream in the lockfile.+ const importedLocalHash = localSkillHash(importRoot, targetName);+ const importedPlugin = inspectPluginSource(sourceLabel);+ // baselineRef only for project scope — global is unversioned (no commit above).+ const importedBaselineRef =+ scope === 'project'+ ? await deps.shadowHeadSha(deps.artifactWriterId(actor), `${importHomeRel}/${targetName}`)+ : undefined;+ // Re-read inside the serialized mutation: `lock` above was snapshotted before+ // the fetch, and writing that snapshot back would erase any entry a+ // concurrent import added while this one was cloning.+ await mutateSkillsLock(lockPath, (current) =>+ upsertLockEntry(current, targetName, {+ source: sourceLabel,+ ...(importedPlugin ? { pluginProvider: importedPlugin.provider } : {}),+ ref,+ contentHash: acquired.contentHash,+ files: acquired.files.map((file) => file.relPath),+ ...(importedLocalHash !== undefined ? { localHash: importedLocalHash } : {}),+ ...(importedBaselineRef !== undefined ? { baselineRef: importedBaselineRef } : {}),+ publisher,+ importedAt: new Date().toISOString(),+ ...(upstreamSkill !== undefined ? { skill: upstreamSkill } : {}),+ }),+ );++ // Fan the freshly-imported IN-PLACE skill out to the configured editors so+ // it is live in Claude/Cursor/Codex on import (guarded copy/link+ // primitives — never clobbers a differing dir). Best-effort: a projection+ // failure must not fail the import. Skipped when the caller installs+ // explicitly afterwards (`install: false`). Copies auto-pair into the+ // placements ledger at the next re-sync, so forward refresh covers them.+ if (params.skipProjection !== true) {+ try {+ const targets = resolveSkillTargets(importBase);+ if (targets.length > 0) {+ const canonicalAbs = resolve(importRoot, targetName);+ // A re-import lands on a skill that may already occupy editor dirs as+ // copies; only a genuinely fresh one takes the symlink default.+ const projectionRoots = skillProjectionRoots(scope);+ // A re-import can land on dests that already exist; those that are+ // symlinks keep the symlink default, matching what is already there.+ const occupied = targets.filter((editor) => {+ const dest = skillHostDir(importBase, editor, targetName, projectionRoots);+ return dest !== null && dest !== canonicalAbs && existsSync(dest);+ });+ const occupiedLinks = occupied.filter((editor) => {+ const dest = skillHostDir(importBase, editor, targetName, projectionRoots);+ try {+ return dest !== null && lstatSync(dest).isSymbolicLink();+ } catch {+ return false;+ }+ });+ projectInPlaceSkill({+ canonicalAbs,+ canonicalHash: parseSkillDir(canonicalAbs)?.contentHash ?? '',+ canonicalRootRel: importHomeRel,+ name: targetName,+ cwd: importBase,+ targets,+ mode: deps.effectiveInstallMode(scope, targetName, {+ hosts: ['<source>', ...occupied],+ linkedHosts: occupiedLinks,+ }),+ roots: skillProjectionRoots(scope),+ });+ }+ } catch (err) {+ log.warn({ err, skill: targetName }, 'post-import editor fan-out failed (import kept)');+ }+ }++ deps.signalFiles();+ return {+ ok: true,+ body: {+ name: targetName,+ path: `${importHomeRel}/${targetName}/SKILL.md`,+ created: wr.created,+ alreadyImported: false,+ ...(collided ? { collisionRenamedFrom: acquired.name } : {}),+ provenance: { source: sourceLabel, ref, contentHash: acquired.contentHash, publisher },+ warnings,+ },+ };+ },+ };+}diff --git a/packages/server/src/services/skill-install-ops.ts b/packages/server/src/services/skill-install-ops.tsnew file mode 100644index 000000000..4b780e240--- /dev/null+++ b/packages/server/src/services/skill-install-ops.ts@@ -0,0 +1,733 @@+import { existsSync, lstatSync, readFileSync, realpathSync } from 'node:fs';+import { dirname, join, relative, resolve, sep } from 'node:path';+import {+ applyPatchToFm,+ detectFmRegion,+ EDITOR_PROJECT_SKILL_ROOT,+ EDITOR_USER_SKILL_ROOT,+ type EditorId,+ isSkillInstallTarget,+ SKILL_NAME_REGEX,+ type SkillInstallWarningCode,+} from '@inkeep/open-knowledge-core';+import { parseSkillDir, type SkillHostId } from '@inkeep/open-knowledge-core/skills-catalog';+import {+ tracedCpSync,+ tracedMkdirSync,+ tracedRenameSync,+ tracedRmSync,+ tracedSymlinkSync,+ tracedWriteFileSync,+} from '../fs-traced.ts';+import {+ scanGlobalInPlaceSkills,+ scanHostRootAliases,+ scanInPlaceSkills,+} from '../in-place-skills.ts';+import { removeSkillInstall } from '../installed-skills-marker.ts';+import { getLogger } from '../logger.ts';+import { unlinkEditorSkillFolder } from '../skill-folder-links.ts';+import {+ isRefusedOkPlacementRoot,+ readSkillPlacements,+ recordFolderExpectation,+ recordSkillPlacement,+ recordSkillSourceHost,+ removeSkillPlacement,+ resolveSkillPlacementPath,+} from '../skill-placements.ts';+import {+ classifyInPlaceDest,+ projectInPlaceSkill,+ relocateInPlaceCanonical,+ removeInPlaceSkillCopies,+ skillProjectionRoots,+} from '../skill-projection.ts';++/**+ * Skill-install verbs that reshape a skill's real locations (as opposed to+ * the placement ledger verbs in skill-placement-ops.ts). First resident:+ * fork resolution — same name, different bytes in a non-canonical editor+ * dir. Every path stashes the bytes it discards to+ * `~/.ok/edit-backups/forks/` first — resolution is never silently lossy.+ */++const log = getLogger('skill-install');++/** The canonical in-place entry a fork is resolved against. */+interface ForkCanonical {+ dir: string;+ contentHash: string;+ hosts: readonly string[];+}++type ResolveForkOutcome =+ | { ok: true; warnings: string[] }+ | { ok: false; kind: 'unknown-editor'; editor: string }+ | { ok: false; kind: 'fork-absent' }+ | { ok: false; kind: 'not-a-fork' }+ | { ok: false; kind: 'invalid-new-name'; toName: string }+ | { ok: false; kind: 'name-taken'; toName: string };++type ApplyAddRemoveOutcome =+ | { ok: true; targets: string[]; rootAdds: string[]; rootRemoves: string[] }+ | { ok: false; kind: 'remove-source'; sourceId: string }+ | { ok: false; kind: 'unfollow-failed'; subRoot: string; reason: string };++type PromoteSourceOutcome =+ | { ok: true; hosts: string[]; sourceMovedTo?: string }+ | { ok: false; kind: 'invalid-target'; target: string }+ | { ok: false; kind: 'source-occupied'; reason: string };++type FanOutInPlaceOutcome =+ | {+ ok: true;+ hosts: string[];+ warnings: string[];+ warningCodes: SkillInstallWarningCode[];+ sourceMovedTo?: string;+ }+ | { ok: false; kind: 'source-occupied'; reason: string };++export interface SkillInstallOpsDeps {+ contentDir: string;+ skillsHome: string;+}++export interface SkillInstallOpsService {+ resolveFork(input: {+ scope: 'project' | 'global';+ name: string;+ fork: { editor: string; action: 'align' | 'make-source' | 'rename'; toName?: string };+ inPlaceEntry: ForkCanonical;+ }): ResolveForkOutcome;+ /**+ * Additive `add`/`remove` (stateless location callers): translate into the+ * set-exact host math + custom-root placement lists, applying the alias+ * materialize/unfollow remedies so one verb means "this agent gets / stops+ * getting the skill" however its folder is wired.+ */+ applyAddRemove(input: {+ scope: 'project' | 'global';+ name: string;+ inPlaceEntry: ForkCanonical;+ add?: readonly string[];+ remove?: readonly string[];+ }): Promise<ApplyAddRemoveOutcome>;+ /**+ * SOURCE promotion for a STORE-BACKED skill: the chosen host's location+ * becomes the real folder, sibling symlinks re-point, the install-marker+ * entry drops (the in-place scan is truth from here on), and the choice is+ * sticky.+ */+ promoteStoreBackedSource(input: {+ scope: 'project' | 'global';+ name: string;+ base: string;+ skillDir: string;+ newSource: SkillHostId;+ }): Promise<PromoteSourceOutcome>;+ /**+ * One-shot SOURCE move for an IN-PLACE skill: relocate the real folder to+ * the chosen host (editor id, the hub, or a custom skills root) and leave a+ * symlink at the old source path — set-source is never a removal.+ */+ promoteInPlaceSource(input: {+ scope: 'project' | 'global';+ name: string;+ base: string;+ prefBase?: string;+ skillDir: string;+ inPlaceEntry: ForkCanonical;+ newSource: string;+ }): Promise<PromoteSourceOutcome>;+ /**+ * Set-exact fan-out for an IN-PLACE skill: relocate the source when it was+ * explicitly unchecked, strip lossless copies the target set no longer+ * names, project into newly-checked hosts, convert named custom placements+ * to the requested form, record receipts, and apply additive custom-root+ * adds/removes. Returns the honest post-op host set from a re-scan.+ */+ fanOutInPlace(input: {+ scope: 'project' | 'global';+ name: string;+ /** cwd host dirs resolve against; also where the install marker lives. */+ base: string;+ /** Placement-ledger base (per-skill preference scope); absent in harnesses. */+ prefBase?: string;+ skillDir: string;+ inPlaceEntry: ForkCanonical;+ canonicalRootRel: string;+ inPlaceTargets: SkillHostId[];+ /** True when the caller named an explicit target set (set-exact semantics). */+ setExact: boolean;+ installMode: 'copy' | 'link';+ linkModeReq?: boolean;+ rootAdds: string[];+ rootRemoves: string[];+ }): Promise<FanOutInPlaceOutcome>;+}++export function createSkillInstallOpsService(deps: SkillInstallOpsDeps): SkillInstallOpsService {+ const scanScope = (scope: 'project' | 'global') =>+ scope === 'project'+ ? scanInPlaceSkills(deps.contentDir)+ : scanGlobalInPlaceSkills(deps.skillsHome);++ return {+ resolveFork(input) {+ const { scope, name, fork, inPlaceEntry } = input;+ const inPlaceScanBase = scope === 'project' ? deps.contentDir : deps.skillsHome;+ // Warnings raised while resolving a fork — surfaced on the response so a+ // half-completed rename is not reported as a clean success.+ const forkWarnings: string[] = [];+ const rootMap = scope === 'project' ? EDITOR_PROJECT_SKILL_ROOT : EDITOR_USER_SKILL_ROOT;+ const forkRootRel =+ fork.editor === 'agents' ? '.agents/skills' : (rootMap[fork.editor as EditorId] ?? null);+ if (forkRootRel === null) {+ return { ok: false, kind: 'unknown-editor', editor: fork.editor };+ }+ const forkDir = resolve(inPlaceScanBase, forkRootRel, name);+ const canonicalAbs = resolve(inPlaceScanBase, inPlaceEntry.dir);+ const forkParsed = parseSkillDir(forkDir);+ if (forkParsed === null || forkDir === canonicalAbs) {+ return { ok: false, kind: 'fork-absent' };+ }+ if (forkParsed.contentHash === inPlaceEntry.contentHash) {+ return { ok: false, kind: 'not-a-fork' };+ }+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');+ const stash = (dirAbs: string, label: string): void => {+ try {+ const dest = resolve(+ deps.skillsHome,+ '.ok',+ 'edit-backups',+ 'forks',+ `${name}-${label}-${stamp}`,+ );+ tracedMkdirSync(resolve(dest, '..'), { recursive: true });+ tracedCpSync(dirAbs, dest, { recursive: true });+ } catch (e) {+ log.warn({ name, err: e }, '[skill-fork] stash failed');+ }+ };+ if (fork.action === 'align') {+ // The fork loses: stash it, remove it, re-project the canonical.+ stash(forkDir, fork.editor);+ tracedRmSync(forkDir, { recursive: true, force: true });+ projectInPlaceSkill({+ canonicalAbs,+ canonicalHash: inPlaceEntry.contentHash,+ canonicalRootRel: dirname(inPlaceEntry.dir),+ name,+ cwd: inPlaceScanBase,+ targets: [fork.editor as SkillHostId].filter(isSkillInstallTarget),+ roots: skillProjectionRoots(scope),+ });+ } else if (fork.action === 'make-source') {+ // The fork wins: stash + remove the old canonical and its same-hash+ // copies; the fork's dir is then the only real dir and the scan+ // elects it; re-project to the hosts the skill had.+ const oldHosts = inPlaceEntry.hosts.filter(isSkillInstallTarget);+ stash(canonicalAbs, 'source');+ removeInPlaceSkillCopies({+ canonicalAbs,+ canonicalHash: inPlaceEntry.contentHash,+ name,+ cwd: inPlaceScanBase,+ targets: oldHosts,+ roots: skillProjectionRoots(scope),+ });+ tracedRmSync(canonicalAbs, { recursive: true, force: true });+ const rescanned = scanScope(scope).find((sk) => sk.name === name);+ if (rescanned) {+ projectInPlaceSkill({+ canonicalAbs: resolve(inPlaceScanBase, rescanned.dir),+ canonicalHash: rescanned.contentHash,+ canonicalRootRel: dirname(rescanned.dir),+ name,+ cwd: inPlaceScanBase,+ targets: oldHosts,+ roots: skillProjectionRoots(scope),+ });+ }+ } else {+ // Keep both: the fork's dir becomes an independent skill under a+ // NEW name (frontmatter rewritten in lock-step with the dir).+ const toName = fork.toName as string;+ if (!SKILL_NAME_REGEX.test(toName)) {+ return { ok: false, kind: 'invalid-new-name', toName };+ }+ if (scanScope(scope).some((sk) => sk.name === toName)) {+ return { ok: false, kind: 'name-taken', toName };+ }+ const dest = resolve(inPlaceScanBase, forkRootRel, toName);+ tracedRenameSync(forkDir, dest);+ // The folder move already happened. If the frontmatter patch below+ // fails the skill is left half-renamed, which the caller has to+ // hear about — a 200 with no warnings reads as a clean rename.+ const skillMdPath = resolve(dest, 'SKILL.md');+ try {+ // Patch the YAML region, not the file. A multiline `^name:` regex+ // matches the first such line ANYWHERE, so a body line reading+ // `name: …` (a code sample, a config snippet) got rewritten+ // instead of the frontmatter. Same primitives the duplicate+ // handler uses.+ const raw = readFileSync(skillMdPath, 'utf-8');+ const { fenced, body: skillBody } = detectFmRegion(raw);+ // presence-exempt: no CRDT write, no agent identity+ const renamed = applyPatchToFm(fenced, { name: toName });+ if (renamed.ok) {+ tracedWriteFileSync(skillMdPath, `${renamed.nextFenced}${skillBody}`);+ } else {+ log.warn(+ { name: toName, reason: renamed.error.kind },+ '[skill-fork] frontmatter rename failed',+ );+ forkWarnings.push(+ `Renamed the folder to "${toName}", but its SKILL.md still declares the old name — edit the frontmatter to match.`,+ );+ }+ } catch (e) {+ log.warn({ name: toName, err: e }, '[skill-fork] frontmatter rename failed');+ forkWarnings.push(+ `Renamed the folder to "${toName}", but its SKILL.md could not be updated — edit the frontmatter to match.`,+ );+ }+ }+ return { ok: true, warnings: forkWarnings };+ },++ async applyAddRemove(input) {+ const { scope, name, inPlaceEntry } = input;+ const inPlaceScanBase = scope === 'project' ? deps.contentDir : deps.skillsHome;+ const rootAdds: string[] = [];+ const rootRemoves: string[] = [];+ const stripHome = (id: string) => id.replace(/\\/g, '/').replace(/^~\//, '');+ const sourceId = inPlaceEntry.hosts[0];+ // The source is untouchable here — a skill's source folder IS the+ // skill; removing it is `delete`, moving it is `source`.+ if (sourceId !== undefined && (input.remove ?? []).some((id) => stripHome(id) === sourceId)) {+ return { ok: false, kind: 'remove-source', sourceId };+ }+ const hostSet = new Set(inPlaceEntry.hosts.filter((h) => isSkillInstallTarget(h)));+ for (const id of input.add ?? []) {+ if (isSkillInstallTarget(id)) hostSet.add(id);+ else rootAdds.push(stripHome(id));+ }+ // A removed host that is NOT a physical location but READS the skill+ // through a folder alias (its skills folder symlinked into a pool+ // root holding the skill) gets the unfollow remedy automatically:+ // its folder unlinks from the pool EXCLUDING this skill — one verb+ // means "this agent stops getting the skill" however it got it.+ const aliasMap = scanHostRootAliases(inPlaceScanBase, scope);+ const aliasUnfollows: string[] = [];+ // The ADD-side mirror: an added host whose skills folder is an alias+ // into a pool that does NOT hold this skill would otherwise be a+ // silent no-op (projection never writes through an alias). One verb+ // means "this agent gets the skill" however its folder is wired — so+ // the alias unlinks into per-skill links first (keeping everything it+ // followed), and the projection below then writes the real bundle.+ const aliasMaterializes: string[] = [];+ for (const id of input.add ?? []) {+ if (+ isSkillInstallTarget(id) &&+ aliasMap[id] !== undefined &&+ !existsSync(resolve(inPlaceScanBase, aliasMap[id], name, 'SKILL.md'))+ ) {+ const subRoot =+ id === 'agents'+ ? '.agents/skills'+ : ((scope === 'project' ? EDITOR_PROJECT_SKILL_ROOT : EDITOR_USER_SKILL_ROOT)[+ id as EditorId+ ] ?? null);+ if (subRoot !== null) aliasMaterializes.push(subRoot);+ }+ }+ for (const id of input.remove ?? []) {+ if (isSkillInstallTarget(id)) {+ if (+ !hostSet.has(id) &&+ aliasMap[id] !== undefined &&+ existsSync(resolve(inPlaceScanBase, aliasMap[id], name, 'SKILL.md'))+ ) {+ const subRoot =+ id === 'agents'+ ? '.agents/skills'+ : ((scope === 'project' ? EDITOR_PROJECT_SKILL_ROOT : EDITOR_USER_SKILL_ROOT)[+ id as EditorId+ ] ?? null);+ if (subRoot !== null) aliasUnfollows.push(subRoot);+ }+ hostSet.delete(id);+ } else rootRemoves.push(stripHome(id));+ }+ for (const { subRoot, exclude } of [+ ...aliasMaterializes.map((subRoot) => ({ subRoot, exclude: undefined })),+ ...aliasUnfollows.map((subRoot) => ({ subRoot, exclude: [name] })),+ ]) {+ const r = unlinkEditorSkillFolder({+ base: inPlaceScanBase,+ folderRel: subRoot,+ ...(exclude !== undefined ? { exclude } : {}),+ });+ if (r.ok) await recordFolderExpectation(inPlaceScanBase, subRoot, { expect: 'own' });+ if (!r.ok) {+ return { ok: false, kind: 'unfollow-failed', subRoot, reason: r.reason };+ }+ }+ return { ok: true, targets: [...hostSet], rootAdds, rootRemoves };+ },++ async promoteStoreBackedSource(input) {+ const { scope, name, base, skillDir, newSource } = input;+ const inPlaceScanBase = scope === 'project' ? deps.contentDir : deps.skillsHome;+ const moved = relocateInPlaceCanonical({+ canonicalAbs: skillDir,+ canonicalHash: parseSkillDir(skillDir)?.contentHash ?? '',+ name,+ cwd: inPlaceScanBase,+ newTarget: newSource,+ roots: skillProjectionRoots(scope),+ });+ if (!moved.ok) {+ return { ok: false, kind: 'source-occupied', reason: moved.reason };+ }+ await removeSkillInstall(base, name);+ await recordSkillSourceHost(inPlaceScanBase, name, newSource);+ const postPromote = scanScope(scope).find((s) => s.name === name);+ return {+ ok: true,+ hosts: postPromote ? [...postPromote.hosts] : [],+ sourceMovedTo: relative(inPlaceScanBase, moved.newAbs).split(sep).join('/'),+ };+ },++ async promoteInPlaceSource(input) {+ const { scope, name, base, prefBase, skillDir, inPlaceEntry, newSource } = input;+ const inPlaceScanBase = scope === 'project' ? deps.contentDir : deps.skillsHome;+ const ledgerBase = prefBase ?? base;+ const currentSource = inPlaceEntry.hosts[0] as string | undefined;+ // Editor ids + the hub map to host dirs; anything else is a CUSTOM+ // skills-root path (its synthetic host id IS the root path).+ const isHostTarget = isSkillInstallTarget(newSource);+ let customDestAbs: string | undefined;+ if (!isHostTarget) {+ const rel = newSource+ .replace(/\\/g, '/')+ .replace(/^~\//, '')+ .replace(/^\/+|\/+$/g, '');+ const invalidOk = isRefusedOkPlacementRoot(rel);+ const destAbs = resolve(base, rel, name);+ if (rel === '' || invalidOk || !destAbs.startsWith(resolve(base) + sep)) {+ return { ok: false, kind: 'invalid-target', target: newSource };+ }+ customDestAbs = destAbs;+ }+ let sourceMovedTo: string | undefined;+ if (currentSource !== undefined && newSource !== currentSource) {+ const oldSourceRel = relative(inPlaceScanBase, resolve(skillDir)).split(sep).join('/');+ const moved = relocateInPlaceCanonical({+ canonicalAbs: skillDir,+ canonicalHash: inPlaceEntry.contentHash,+ name,+ cwd: base,+ newTarget: (isHostTarget ? newSource : 'agents') as SkillHostId,+ ...(customDestAbs !== undefined ? { destDirAbs: customDestAbs } : {}),+ // The promote/downgrade SWAP: the old source path becomes a+ // symlink to the new one (set-source is never a removal).+ leaveLinkBehind: true,+ roots: skillProjectionRoots(scope),+ });+ if (!moved.ok) {+ return { ok: false, kind: 'source-occupied', reason: moved.reason };+ }+ sourceMovedTo = relative(inPlaceScanBase, moved.newAbs).split(sep).join('/');+ // The swap left a symlink at the old source path (relocate's+ // `leaveLinkBehind`). RECORD it as the expected form — without+ // the receipt, an external tool rewriting that link as a copy+ // would be invisible to drift detection (the .agents blind spot).+ await recordSkillPlacement(ledgerBase, name, {+ path: oldSourceRel,+ mode: 'link',+ });+ // Re-point recorded placement SYMLINKS that referenced the old+ // source (the relocation loop only covers host dirs).+ for (const p of readSkillPlacements(ledgerBase)[name] ?? []) {+ const abs = resolve(ledgerBase, p.path);+ if (abs === moved.newAbs) continue;+ try {+ if (!lstatSync(abs).isSymbolicLink()) continue;+ } catch {+ continue;+ }+ let target: string | null = null;+ try {+ target = realpathSync(abs);+ } catch {+ target = null; // dangling (pointed at the moved-away dir)+ }+ if (target === null || target === resolve(skillDir)) {+ tracedRmSync(abs, { recursive: true, force: true });+ tracedSymlinkSync(relative(dirname(abs), moved.newAbs), abs, 'dir');+ }+ }+ // The promoted path is now the REAL dir — refresh its receipt to+ // the promoted form (hash-less copy: no drift, resync skips it,+ // and a custom root stays registered with the scan). A leftover+ // 'link' expectation would misread the promotion as drift+ // (OVERWRITTEN on a folder OK itself just wrote), inviting an+ // accidental hands-off.+ await recordSkillPlacement(ledgerBase, name, {+ path: sourceMovedTo,+ mode: 'copy',+ });+ await recordSkillSourceHost(ledgerBase, name, newSource);+ }+ const postMove = scanScope(scope).find((sk) => sk.name === name);+ return {+ ok: true,+ hosts: postMove ? [...postMove.hosts] : [...inPlaceEntry.hosts],+ ...(sourceMovedTo !== undefined ? { sourceMovedTo } : {}),+ };+ },++ async fanOutInPlace(input) {+ const { scope, name, base, prefBase, inPlaceEntry, inPlaceTargets } = input;+ const inPlaceScanBase = scope === 'project' ? deps.contentDir : deps.skillsHome;+ const ledgerBase = prefBase ?? base;+ let { skillDir } = input;+ const warnings: string[] = [];+ const warningCodes: SkillInstallWarningCode[] = [];++ const prior = inPlaceEntry.hosts.filter((h): h is SkillHostId => isSkillInstallTarget(h));+ // Unchecking THE SOURCE relocates it: the bundle moves to the+ // highest-precedence remaining target (its copy/link there becomes+ // the real dir), sibling links re-point, and set-exact proceeds+ // against the new source. Refused when no other target remains —+ // removing the only location is delete, not uninstall.+ // `targets: []` (uninstall everywhere) is NOT a relocation request:+ // lossless copies strip below and the source survives, matching the+ // never-delete invariant.+ // A CUSTOM-ROOT source (path-like host id, e.g. `.ok/skills`) is+ // inexpressible in the editor-checkbox target vocabulary, so its+ // absence from a set-exact list is NOT an uncheck — a mode flip+ // (Copies/Symlinks re-apply) must never relocate it. Only an+ // editor/hub source can be unchecked, and only that relocates.+ // Relocation needs an EXPLICIT uncheck. With `targets` omitted the+ // caller expressed no opinion about membership at all — the list came+ // from `resolveSkillTargets`, which never includes the `agents` hub, so+ // a defaults-driven call would read the hub's absence as an uncheck and+ // move the real folder out from under it. Same reasoning as the two+ // carve-outs above: `targets: []` and a custom-root source are excluded+ // because neither is an uncheck either.+ const sourceHost = inPlaceEntry.hosts[0] as SkillHostId | undefined;+ let sourceMovedTo: string | undefined;+ if (+ input.setExact &&+ sourceHost !== undefined &&+ !sourceHost.includes('/') &&+ inPlaceTargets.length > 0 &&+ !inPlaceTargets.includes(sourceHost)+ ) {+ const newTarget = inPlaceTargets[0] as SkillHostId;+ const moved = relocateInPlaceCanonical({+ canonicalAbs: skillDir,+ canonicalHash: inPlaceEntry.contentHash,+ name,+ cwd: base,+ newTarget,+ roots: skillProjectionRoots(scope),+ });+ if (!moved.ok) {+ return { ok: false, kind: 'source-occupied', reason: moved.reason };+ }+ skillDir = moved.newAbs;+ sourceMovedTo = relative(inPlaceScanBase, moved.newAbs).split(sep).join('/');+ // The path holding the source is now the REAL dir — a stale 'link'+ // receipt there would misread the relocation as drift.+ await recordSkillPlacement(ledgerBase, name, {+ path: sourceMovedTo,+ mode: 'copy',+ });+ // Sticky: without this the next scan re-elects the source by+ // static precedence and the user's choice silently reverts.+ await recordSkillSourceHost(ledgerBase, name, newTarget);+ }+ const effectiveRootRel = sourceMovedTo+ ? sourceMovedTo.split('/').slice(0, -1).join('/')+ : input.canonicalRootRel;++ removeInPlaceSkillCopies({+ canonicalAbs: skillDir,+ canonicalHash: inPlaceEntry.contentHash,+ name,+ cwd: base,+ // Set-exact removal only when the caller actually named a set. With+ // `targets` omitted this call is additive ("make sure it is in my+ // configured editors"), so subtracting the locations that list+ // happens not to mention would delete work nobody asked to remove.+ targets: input.setExact ? prior.filter((h) => !inPlaceTargets.includes(h)) : [],+ roots: skillProjectionRoots(scope),+ });+ const fanned = projectInPlaceSkill({+ canonicalAbs: skillDir,+ canonicalHash: inPlaceEntry.contentHash,+ canonicalRootRel: effectiveRootRel,+ name,+ cwd: base,+ targets: inPlaceTargets,+ mode: input.installMode,+ // An EXPLICIT copy choice converts existing links back to copies+ // (lossless — the link's bytes ARE the canonical's); an implicit+ // copy default never touches links.+ convertLinks: input.linkModeReq === false,+ roots: skillProjectionRoots(scope),+ });+ for (const editor of fanned.conflicted) {+ warnings.push(+ `A different skill named "${name}" already exists in the ${editor} skills folder — left untouched.`,+ );+ warningCodes.push('name-conflict');+ }+ // Custom placements the call is ADDING adopt the requested form; ones+ // it does not name are left alone. A blanket flip of every recorded+ // placement as a side effect of installing is exactly what the app+ // retired — converting is its own verb, per location, and asks first.+ if (input.linkModeReq !== undefined && prefBase && input.rootAdds.length > 0) {+ const adding = new Set(input.rootAdds.map((r) => `${r}/${name}`));+ for (const p of (readSkillPlacements(prefBase)[name] ?? []).filter((p) =>+ adding.has(p.path),+ )) {+ const abs = resolve(prefBase, p.path);+ if (abs === resolve(skillDir)) continue;+ const cls = classifyInPlaceDest(abs, resolve(skillDir), inPlaceEntry.contentHash);+ if (input.linkModeReq === true && cls === 'same-copy') {+ tracedRmSync(abs, { recursive: true, force: true });+ tracedSymlinkSync(relative(dirname(abs), resolve(skillDir)), abs, 'dir');+ await recordSkillPlacement(prefBase, name, { path: p.path, mode: 'link' });+ } else if (input.linkModeReq === false && cls === 'link-to-canonical') {+ tracedRmSync(abs, { recursive: true, force: true });+ tracedCpSync(resolve(skillDir), abs, { recursive: true, dereference: true });+ await recordSkillPlacement(prefBase, name, {+ path: p.path,+ mode: 'copy',+ hash: inPlaceEntry.contentHash,+ });+ }+ }+ }++ // Record what we just WROTE (machine-local): copies so the forward+ // re-sync can lossless-refresh them, links as the EXPECTED form so+ // drift detection can spot another tool rewriting the path.+ for (const editor of fanned.hosts) {+ const editorRoot =+ editor === 'agents' ? '.agents/skills' : EDITOR_PROJECT_SKILL_ROOT[editor];+ if (editorRoot === null || editorRoot === input.canonicalRootRel) continue;+ const copyAbs = resolve(base, editorRoot, name);+ let isLink = false;+ try {+ isLink = lstatSync(copyAbs).isSymbolicLink();+ } catch {+ continue;+ }+ if (isLink) {+ if (input.installMode === 'link') {+ await recordSkillPlacement(ledgerBase, name, {+ path: `${editorRoot}/${name}`,+ mode: 'link',+ });+ }+ continue;+ }+ const copyHash = existsSync(join(copyAbs, 'SKILL.md'))+ ? parseSkillDir(copyAbs)?.contentHash+ : undefined;+ if (copyHash !== undefined && copyHash === inPlaceEntry.contentHash) {+ await recordSkillPlacement(ledgerBase, name, {+ path: `${editorRoot}/${name}`,+ mode: 'copy',+ hash: copyHash,+ });+ }+ }+ // ── Additive custom-root placements (path-like `add`/`remove`+ // members). Idempotent: an already-satisfied add is a no-op; a+ // hand-edited fork is refused with a warning, never deleted.+ if ((input.rootAdds.length > 0 || input.rootRemoves.length > 0) && prefBase) {+ const canonAbs = resolve(skillDir);+ for (const rootRel of input.rootAdds) {+ const underOk = isRefusedOkPlacementRoot(rootRel);+ const parentAbs = resolve(prefBase, rootRel);+ // Symlink-aware containment, the same resolver `place` / `unplace`+ // / `convert` use. A lexical `startsWith` prefix test walks+ // straight through a checked-in symlink (`.team ->+ // ~/Library/LaunchAgents`), and the mkdir + cp below would then+ // write the bundle outside the project entirely.+ const destAbs = resolveSkillPlacementPath(prefBase, `${rootRel}/${name}`);+ if (rootRel === '' || underOk || destAbs === null || destAbs === canonAbs) {+ warnings.push(`"${rootRel}" is not a placeable custom root — skipped.`);+ warningCodes.push('place-path-invalid');+ continue;+ }+ const cls = classifyInPlaceDest(destAbs, canonAbs, inPlaceEntry.contentHash);+ if (cls === 'different') {+ warnings.push(`A different "${name}" already exists at ${rootRel} — left untouched.`);+ warningCodes.push('name-conflict');+ continue;+ }+ if (cls === 'absent') {+ tracedMkdirSync(parentAbs, { recursive: true });+ if (input.installMode === 'link') {+ tracedSymlinkSync(relative(parentAbs, canonAbs), destAbs, 'dir');+ } else {+ tracedCpSync(canonAbs, destAbs, { recursive: true, dereference: true });+ }+ }+ await recordSkillPlacement(prefBase, name, {+ path: `${rootRel}/${name}`,+ mode: input.installMode,+ ...(input.installMode === 'copy' ? { hash: inPlaceEntry.contentHash } : {}),+ });+ }+ for (const rootRel of input.rootRemoves) {+ const rel = `${rootRel}/${name}`;+ const recorded = readSkillPlacements(prefBase)[name]?.find((pl) => pl.path === rel);+ const absDir = resolveSkillPlacementPath(prefBase, rel);+ if (absDir === null) {+ warnings.push(`"${rootRel}" is not a placeable custom root — skipped.`);+ warningCodes.push('place-path-invalid');+ continue;+ }+ const cls = classifyInPlaceDest(absDir, canonAbs, inPlaceEntry.contentHash);+ if (cls === 'different') {+ warnings.push(+ `The copy at ${rel} has been hand-edited (a fork) — refused, never deleted. Remove it manually if you mean it.`,+ );+ warningCodes.push('place-fork-refused');+ continue;+ }+ if (cls !== 'absent' && cls !== 'canonical-dir') {+ tracedRmSync(absDir, { recursive: true, force: true });+ }+ if (recorded) await removeSkillPlacement(prefBase, name, rel);+ }+ }++ // Report the FULL honest post-op host set by re-scanning — covers+ // hub adds/removes, canonical-protection skips, and capability-covered+ // hosts, so the client's before/after diff never invents changes.+ const postEntry = scanScope(scope).find((s) => s.name === name);+ const hosts = postEntry ? [...postEntry.hosts] : fanned.hosts;+ return {+ ok: true,+ hosts,+ warnings,+ warningCodes,+ ...(sourceMovedTo !== undefined ? { sourceMovedTo } : {}),+ };+ },+ };+}diff --git a/packages/server/src/services/skill-placement-ops.ts b/packages/server/src/services/skill-placement-ops.tsnew file mode 100644index 000000000..63e3ac80e--- /dev/null+++ b/packages/server/src/services/skill-placement-ops.ts@@ -0,0 +1,215 @@+import { existsSync } from 'node:fs';+import { dirname, relative, resolve, sep } from 'node:path';+import type { EditorId } from '@inkeep/open-knowledge-core';+import { isSkillInstallTarget } from '@inkeep/open-knowledge-core';+import { parseSkillDir } from '@inkeep/open-knowledge-core/skills-catalog';+import { tracedCpSync, tracedMkdirSync, tracedRmSync, tracedSymlinkSync } from '../fs-traced.ts';+import {+ isRefusedOkPlacementRoot,+ readSkillPlacements,+ recordSkillPlacement,+ removeSkillPlacement,+ resolveSkillPlacementPath,+} from '../skill-placements.ts';+import { classifyInPlaceDest, skillProjectionRoots } from '../skill-projection.ts';++/**+ * One-shot custom skill placements: put a copy or symlink of a bundle under+ * an arbitrary project-relative dir, and the lossless inverse. Placement+ * writes host dirs on this machine, OUTSIDE the content/CRDT plane — the+ * transport maps outcomes and owns the base-resolution (project vs global)+ * plus the CC1 signal.+ */++type PlaceOutcome =+ | { ok: true; placedAt: string }+ /** Asking for the location the skill ALREADY occupies — satisfied, nothing written. */+ | { ok: true; alreadyAtSource: true; placedAt: string }+ | { ok: false; kind: 'invalid-path' }+ | { ok: false; kind: 'dest-exists' };++type UnplaceOutcome =+ | { ok: true }+ | { ok: false; kind: 'not-recorded'; path: string }+ | { ok: false; kind: 'unsafe-path' }+ | { ok: false; kind: 'forked'; path: string }+ | { ok: false; kind: 'canonical-dir'; path: string };++type ConvertOutcome =+ | { ok: true }+ | { ok: false; kind: 'invalid-location' }+ | { ok: false; kind: 'canonical-dir' }+ | { ok: false; kind: 'forked' }+ | { ok: false; kind: 'not-installed' };++export interface SkillPlacementOpsService {+ place(input: {+ placeBase: string;+ name: string;+ /** Raw caller-typed dir; normalized here (backslashes, `~/`, edge slashes). */+ rawDir: string;+ skillDir: string;+ mode: 'link' | 'copy';+ }): Promise<PlaceOutcome>;+ unplace(input: {+ placeBase: string;+ name: string;+ rawPath: string;+ skillDir: string;+ }): Promise<UnplaceOutcome>;+ /**+ * Per-location mode change: make ONE installed location a symlink to the+ * source, or an independent copy again. Lossless-only — a hand-edited copy+ * is a fork and is refused, never overwritten.+ */+ convert(input: {+ ledgerBase: string;+ scope: 'project' | 'global';+ name: string;+ /** Host id (`agents` / editor) or a custom skills-root path. */+ target: string;+ mode: 'link' | 'copy';+ skillDir: string;+ canonicalHash: string;+ }): Promise<ConvertOutcome>;+}++export function createSkillPlacementOpsService(): SkillPlacementOpsService {+ return {+ async place(input) {+ const dirRel = input.rawDir+ .replace(/\\/g, '/')+ .replace(/^~\//, '')+ .replace(/^\/+|\/+$/g, '');+ const parentAbs = resolve(input.placeBase, dirRel);+ const placementRel = `${dirRel}/${input.name}`;+ const destAbs = resolveSkillPlacementPath(input.placeBase, placementRel);+ const baseAbs = resolve(input.placeBase);+ // `.ok/` internals are refused; `.ok/skills` is placeable at both+ // scopes. See `isRefusedOkPlacementRoot` for why that one path is+ // carved out.+ const underOkInternals = isRefusedOkPlacementRoot(dirRel);+ if (+ dirRel === '' ||+ destAbs === null ||+ !destAbs.startsWith(baseAbs + sep) ||+ underOkInternals+ ) {+ return { ok: false, kind: 'invalid-path' };+ }+ // Asking for the location the skill ALREADY occupies is a satisfied+ // request, not a bad path. Since imports land in-place at the+ // `.agents/skills` hub, placing a freshly imported skill there names+ // its own canonical dir — every caller (the hub toggle on a preview,+ // `install --place`, MCP) hit an invalid-path error for a skill that+ // was, in fact, exactly where it was asked to be. Report the location+ // and change nothing; a real copy/symlink here would be self-referential.+ if (destAbs === resolve(input.skillDir)) {+ return { ok: true, alreadyAtSource: true, placedAt: placementRel };+ }+ if (existsSync(destAbs)) {+ return { ok: false, kind: 'dest-exists' };+ }+ tracedMkdirSync(parentAbs, { recursive: true });+ if (input.mode === 'link') {+ tracedSymlinkSync(relative(parentAbs, resolve(input.skillDir)), destAbs, 'dir');+ } else {+ tracedCpSync(input.skillDir, destAbs, { recursive: true, dereference: true });+ }+ await recordSkillPlacement(input.placeBase, input.name, {+ path: placementRel,+ mode: input.mode,+ ...(input.mode === 'copy' ? { hash: parseSkillDir(input.skillDir)?.contentHash } : {}),+ });+ return { ok: true, placedAt: placementRel };+ },++ async unplace(input) {+ const rel = input.rawPath+ .replace(/\\/g, '/')+ .replace(/^~\//, '')+ .replace(/^\/+|\/+$/g, '');+ const recorded = readSkillPlacements(input.placeBase)[input.name]?.find(+ (p) => p.path === rel,+ );+ if (!recorded) {+ return { ok: false, kind: 'not-recorded', path: rel };+ }+ const absDir = resolveSkillPlacementPath(input.placeBase, rel);+ if (absDir === null) {+ return { ok: false, kind: 'unsafe-path' };+ }+ const canonicalHash = parseSkillDir(input.skillDir)?.contentHash ?? '';+ switch (classifyInPlaceDest(absDir, resolve(input.skillDir), canonicalHash)) {+ case 'different':+ return { ok: false, kind: 'forked', path: rel };+ case 'canonical-dir':+ return { ok: false, kind: 'canonical-dir', path: rel };+ case 'absent':+ break; // already gone — just drop the record+ default:+ tracedRmSync(absDir, { recursive: true, force: true });+ }+ await removeSkillPlacement(input.placeBase, input.name, rel);+ return { ok: true };+ },++ async convert(input) {+ const roots = skillProjectionRoots(input.scope);+ const isHost = isSkillInstallTarget(input.target);+ const rootRel = isHost+ ? input.target === 'agents'+ ? '.agents/skills'+ : roots[input.target as EditorId]+ : input.target+ .replace(/\\/g, '/')+ .replace(/^~\//, '')+ .replace(/^\/+|\/+$/g, '');+ const absDir =+ rootRel === null || rootRel === ''+ ? null+ : resolveSkillPlacementPath(input.ledgerBase, `${rootRel}/${input.name}`);+ if (rootRel === null || rootRel === '' || absDir === null) {+ return { ok: false, kind: 'invalid-location' };+ }+ const canonicalAbs = resolve(input.skillDir);+ const cls = classifyInPlaceDest(absDir, canonicalAbs, input.canonicalHash);+ if (cls === 'canonical-dir') {+ return { ok: false, kind: 'canonical-dir' };+ }+ if (cls === 'different') {+ return { ok: false, kind: 'forked' };+ }+ if (cls === 'absent') {+ return { ok: false, kind: 'not-installed' };+ }+ // Already in the requested form — nothing to write (a link-to-somewhere-+ // else still gets re-materialized, same as the fan-out treats it).+ const alreadyRight =+ (input.mode === 'link' && cls === 'link-to-canonical') ||+ (input.mode === 'copy' && cls === 'same-copy');+ if (!alreadyRight) {+ const hostRoot = dirname(absDir);+ tracedRmSync(absDir, { recursive: true, force: true });+ tracedMkdirSync(hostRoot, { recursive: true });+ if (input.mode === 'link') {+ tracedSymlinkSync(relative(hostRoot, canonicalAbs), absDir, 'dir');+ } else {+ tracedCpSync(canonicalAbs, absDir, { recursive: true, dereference: true });+ }+ }+ // Recorded in BOTH branches. When disk is already right the ledger may+ // not be, and a record that disagrees with disk is exactly what renders+ // "changed outside" — skipping the write here used to skip the record+ // too, so a stale record could never be reconciled from the UI+ // (converting again just no-opped). Recording is idempotent when the+ // two agree.+ await recordSkillPlacement(input.ledgerBase, input.name, {+ path: `${rootRel}/${input.name}`,+ mode: input.mode,+ ...(input.mode === 'copy' ? { hash: input.canonicalHash } : {}),+ });+ return { ok: true };+ },+ };+}diff --git a/packages/server/src/services/version-ops.ts b/packages/server/src/services/version-ops.tsnew file mode 100644index 000000000..bf4b0f70b--- /dev/null+++ b/packages/server/src/services/version-ops.ts@@ -0,0 +1,103 @@+import { getLogger } from '../logger.ts';+import { gcRenameLog, getOrLoadRenameLogIndex } from '../rename-log.ts';+import {+ enumerateWipChains,+ SERVICE_WRITER,+ type ShadowHandle,+ saveVersion,+ type WriterIdentity,+} from '../shadow-repo.ts';++/**+ * Save-version checkpoint policy. The transport validates writer shapes and+ * threads identity; this module owns writer resolution — explicit list >+ * agent-scoped > fold-everything — and the checkpoint + rename-log GC+ * sequencing that follows.+ */++const log = getLogger('history');++export interface VersionOpsDeps {+ getCurrentBranch?: () => string | null;+ contentRoot?: string;+}++export interface VersionOpsService {+ saveCheckpoint(+ shadow: ShadowHandle,+ input: {+ /** Pre-validated explicit writers from the request body; [] when absent. */+ explicitWriters: WriterIdentity[];+ /** Agent-scoped writer (MCP checkpoint tool path), when the body carried an agentId. */+ agentWriter?: WriterIdentity;+ /** Normalized checkpoint summary, when present. */+ summary?: string;+ },+ ): Promise<{ checkpointRef: string }>;+}++export function createVersionOpsService(deps: VersionOpsDeps): VersionOpsService {+ return {+ async saveCheckpoint(shadow, input) {+ let writers = input.explicitWriters;+ // True only on the empty-body button path, where `enumerateWipChains`+ // already surfaces EVERY WIP chain — upstream included. That makes the+ // enumerated set the complete fold list, so saveVersion must not re-append+ // the upstream writer (a second rev-parse + a no-op delete on the+ // already-reset ref). The explicit-writers and explicit-agentId paths do+ // not enumerate upstream and keep the default (fold it).+ let foldEnumeratedAll = false;++ // Active branch: the button consolidates the branch the user+ // is on, not a hardcoded 'main'.+ const branch = deps.getCurrentBranch?.() ?? 'main';++ if (writers.length === 0) {+ if (input.agentWriter) {+ // Explicit agentId path (MCP checkpoint tool) — scoped to that agent.+ writers = [input.agentWriter];+ } else {+ // A true empty-body Save Version (the UI button) consolidates ALL+ // non-park WIP chains on the active branch — agent + principal ++ // classified — so the button matches the user's "group everything I've+ // done into a version" mental model. Park-tipped refs hold+ // branch-switch state and are excluded. Falls back to the service+ // writer when there is no WIP activity at all (an empty checkpoint).+ const chains = await enumerateWipChains(shadow, branch);+ const foldable = chains.filter((c) => !c.isPark);+ writers =+ foldable.length > 0+ ? foldable.map((c) => ({+ id: c.writerId,+ name: c.writerId,+ email: `${c.writerId}@openknowledge.local`,+ }))+ : [SERVICE_WRITER];+ foldEnumeratedAll = true;+ }+ }++ const result = await saveVersion(+ shadow,+ deps.contentRoot ?? '.',+ writers,+ branch,+ input.summary,+ foldEnumeratedAll ? { includeUpstream: false } : undefined,+ );++ log.info({ checkpointRef: result.checkpointRef }, 'checkpoint');++ // Rename-log GC trigger: saveVersion deletes WIP refs, which is the+ // largest entry-death cliff. Run reachability sweep (no rebuild —+ // boot already covered that).+ try {+ await gcRenameLog(shadow, getOrLoadRenameLogIndex(shadow.gitDir));+ } catch (err) {+ log.warn({ err }, '[rename-log] post-saveVersion GC failed');+ }++ return { checkpointRef: result.checkpointRef };+ },+ };+}diff --git a/packages/server/src/skill-placements.ts b/packages/server/src/skill-placements.tsindex 1fed806f0..9db87cb3e 100644--- a/packages/server/src/skill-placements.ts+++ b/packages/server/src/skill-placements.ts@@ -12,6 +12,7 @@import { existsSync } from 'node:fs';import { join } from 'node:path';+import { LEGACY_SKILL_STORE_ROOT } from '@inkeep/open-knowledge-core';import { parseSkillDir } from '@inkeep/open-knowledge-core/skills-catalog';import { tracedCpSync, tracedRmSync } from './fs-traced.ts';import { type InPlaceSkill, scanInPlaceSkills } from './in-place-skills.ts';@@ -29,6 +30,21 @@ export {type SkillPlacement,} from './skill-placements-store.ts';+/**+ * Whether a placement root is OK's own state area and therefore refused.+ *+ * `.ok/` holds OK's own state (config, local runtime data, the shadow repo),+ * so a skill may not be placed inside it — with one exception that is not a+ * special case so much as history: `.ok/skills` is the retired store, and+ * projects that predate the retirement still have real skills sitting there.+ * Refusing it would strand them, so it stays placeable at BOTH scopes; a+ * placement there is an ordinary custom root like any other.+ */+export function isRefusedOkPlacementRoot(rootRel: string): boolean {+ const underOk = rootRel === '.ok' || rootRel.startsWith('.ok/');+ return underOk && rootRel !== LEGACY_SKILL_STORE_ROOT;+}+/** Record the user-chosen SOURCE host for a skill (sticky relocation). */export async function recordSkillSourceHost(projectDir: string,