Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): well-form search snippets truncated mid-emoji (#2465)

tim-inkeepbb827385401a
diff --git a/.changeset/surrogate-search-snippets.md b/.changeset/surrogate-search-snippets.md
new file mode 100644
index 000000000..e77eed2ea
--- /dev/null
+++ b/.changeset/surrogate-search-snippets.md
@@ -0,0 +1,5 @@
+---
+'@inkeep/open-knowledge': patch
+---
+
+Fix search snippets splitting an emoji across the truncation boundary, which left a lone UTF-16 surrogate in the JSON-RPC response and caused strict MCP clients (Rust / pydantic parsers) to reject it as invalid UTF-8. Snippets are now well-formed before serialization.
diff --git a/packages/core/src/bridge/bind-frontmatter-doc.ts b/packages/core/src/bridge/bind-frontmatter-doc.ts
index c014b1413..8f9d95287 100644
--- a/packages/core/src/bridge/bind-frontmatter-doc.ts
+++ b/packages/core/src/bridge/bind-frontmatter-doc.ts
@@ -1,3 +1,4 @@
+
import type * as Y from 'yjs';
import type { Err, Ok, Result } from '../config/result.ts';
import { type FrontmatterValidationError, toFrontmatterIssue } from '../frontmatter/errors.ts';
diff --git a/packages/core/src/bridge/bridge-invariant.ts b/packages/core/src/bridge/bridge-invariant.ts
index 005627195..8bfdb40cd 100644
--- a/packages/core/src/bridge/bridge-invariant.ts
+++ b/packages/core/src/bridge/bridge-invariant.ts
@@ -1,3 +1,4 @@
+
import { fnv1aDigest } from './hash-util.ts';
export type BridgeInvariantSite = 'observer-b' | 'persistence' | 'test-harness';
diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts
index 8890f6739..c18c086d5 100644
--- a/packages/core/src/bridge/doc-boundary-space.ts
+++ b/packages/core/src/bridge/doc-boundary-space.ts
@@ -1,3 +1,4 @@
+
import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';
const LEADING_BOUNDARY_RE = /^(?:\r?\n)+/;
diff --git a/packages/core/src/bridge/merge-three-way.ts b/packages/core/src/bridge/merge-three-way.ts
index b0a80fbd9..864bc7245 100644
--- a/packages/core/src/bridge/merge-three-way.ts
+++ b/packages/core/src/bridge/merge-three-way.ts
@@ -37,6 +37,7 @@ function mergeThreeWayImpl(baseline: string, userText: string, agentText: string
return parts.join('\n');
}
+
export type BridgeMergeContentLossSide = 'user' | 'agent';
export type BridgeMergeContentLossWhich = 'substring' | 'order';
diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts
index e67f33b40..592f0b956 100644
--- a/packages/core/src/bridge/normalize.ts
+++ b/packages/core/src/bridge/normalize.ts
@@ -1,3 +1,4 @@
+
const COMMONMARK_ESCAPE_RE = /\\([!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~])/g;
const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/;
diff --git a/packages/core/src/markdown/callout-transformer.ts b/packages/core/src/markdown/callout-transformer.ts
index 83d47404a..a363dfd54 100644
--- a/packages/core/src/markdown/callout-transformer.ts
+++ b/packages/core/src/markdown/callout-transformer.ts
@@ -1,3 +1,4 @@
+
import type { Blockquote, Paragraph, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts
index 193e286b0..47f0d07e4 100644
--- a/packages/core/src/markdown/comment-promoter.ts
+++ b/packages/core/src/markdown/comment-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/dedent-block-jsx-close.ts b/packages/core/src/markdown/dedent-block-jsx-close.ts
index ece5c02a1..b6299a948 100644
--- a/packages/core/src/markdown/dedent-block-jsx-close.ts
+++ b/packages/core/src/markdown/dedent-block-jsx-close.ts
@@ -1,3 +1,4 @@
+
import { findFencedRegions, isInsideFence } from './fence-regions.ts';
const INDENTED_BLOCK_JSX_CLOSE_RE = /^([ ]{1,3})(<\/[A-Z][A-Za-z0-9_]*\s*>)([ \t]*)$/gm;
diff --git a/packages/core/src/markdown/details-accordion-promoter.ts b/packages/core/src/markdown/details-accordion-promoter.ts
index 54774764a..acd398aa6 100644
--- a/packages/core/src/markdown/details-accordion-promoter.ts
+++ b/packages/core/src/markdown/details-accordion-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.ts
index c7518d614..fb5ac8e8a 100644
--- a/packages/core/src/markdown/fence-regions.ts
+++ b/packages/core/src/markdown/fence-regions.ts
@@ -1,3 +1,4 @@
+
const FENCE_RE = /^(`{3,}|~{3,})/gm;
export function findFencedRegions(src: string): Array<[number, number]> {
diff --git a/packages/core/src/markdown/fixtures/index.ts b/packages/core/src/markdown/fixtures/index.ts
index 00f5771ee..6931623c7 100644
--- a/packages/core/src/markdown/fixtures/index.ts
+++ b/packages/core/src/markdown/fixtures/index.ts
@@ -1,3 +1,4 @@
+
import { readFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -8,6 +9,7 @@ function fixturePath(...segments: string[]): string {
return resolve(FIXTURES_DIR, ...segments);
}
+
interface GfmExample {
section: string;
markdown: string;
@@ -17,6 +19,7 @@ export function loadGfmExamples(): GfmExample[] {
return JSON.parse(readFileSync(fixturePath('gfm', 'examples.json'), 'utf8')) as GfmExample[];
}
+
interface MdxCrashEntry {
id: string;
input: string;
@@ -68,6 +71,7 @@ export function loadLargeEmbedFixtures(): LargeEmbedFixture[] {
) as LargeEmbedFixture[];
}
+
export function loadPrd6955Before(): string {
return readFileSync(fixturePath('regression', 'prd-6955-before.md'), 'utf8');
}
@@ -76,6 +80,7 @@ export function loadPrd6955CorruptedTriplicated(): string {
return readFileSync(fixturePath('regression', 'prd-6955-corrupted-triplicated.md'), 'utf8');
}
+
export interface NgPinnedCase {
id: string;
name: string;
@@ -92,6 +97,7 @@ export function loadNgPinnedCases(): NgPinnedCase[] {
) as NgPinnedCase[];
}
+
export function loadLargeRealistic(): string {
return readFileSync(fixturePath('perf', 'large-realistic.md'), 'utf8');
}
diff --git a/packages/core/src/markdown/fixtures/perf/generate.ts b/packages/core/src/markdown/fixtures/perf/generate.ts
index 8b1eb8519..daa3ef82b 100644
--- a/packages/core/src/markdown/fixtures/perf/generate.ts
+++ b/packages/core/src/markdown/fixtures/perf/generate.ts
@@ -1,3 +1,4 @@
+
import { writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -10,6 +11,7 @@ const BASELINE_ONLY_COUNTS = [500, 2500] as const;
const SEED = 0xf1de1117;
+
function mulberry32(seed: number): () => number {
let a = seed >>> 0;
return () => {
@@ -36,6 +38,7 @@ function pickWeighted<T>(rand: () => number, items: readonly [T, number][]): T {
return items[items.length - 1][0];
}
+
const WORDS = [
'lorem',
'ipsum',
@@ -153,6 +156,7 @@ function mdxBlock(rand: () => number): string {
return `<${name}>\n\n${sentence(rand)}\n\n</${name}>`;
}
+
type BlockKind = 'paragraph' | 'heading' | 'list' | 'code' | 'table' | 'mdx';
const MIX: readonly [BlockKind, number][] = [
@@ -191,6 +195,7 @@ function generateDocument(blockCount: number, seed: number): string {
return `${blocks.join('\n\n')}\n`;
}
+
function main(): void {
for (const count of [...BLOCK_COUNTS, ...BASELINE_ONLY_COUNTS]) {
const doc = generateDocument(count, SEED);
diff --git a/packages/core/src/markdown/guard-flanking-matrix.ts b/packages/core/src/markdown/guard-flanking-matrix.ts
index 420dc0fe5..32c055c5b 100644
--- a/packages/core/src/markdown/guard-flanking-matrix.ts
+++ b/packages/core/src/markdown/guard-flanking-matrix.ts
@@ -7,6 +7,7 @@ import {
import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts';
import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts';
+
export const ATTENTION_DELIMITERS = ['*', '_', '**', '~~', '=='] as const;
export type FlankClass = 'whitespace' | 'punctuation' | 'other';
diff --git a/packages/core/src/markdown/handler-shadow-audit.ts b/packages/core/src/markdown/handler-shadow-audit.ts
index 2653473da..283913255 100644
--- a/packages/core/src/markdown/handler-shadow-audit.ts
+++ b/packages/core/src/markdown/handler-shadow-audit.ts
@@ -1,3 +1,4 @@
+
export interface HandlerShadowWitness {
input: string;
expect: 'byte' | 'byte-and-reparse-type';
diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.ts
index f758cf737..7a22e8016 100644
--- a/packages/core/src/markdown/highlight-promoter.ts
+++ b/packages/core/src/markdown/highlight-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Parent, PhrasingContent, Root, Text } from 'mdast';
import type { MdxJsxTextElement } from 'mdast-util-mdx';
import { SKIP, visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.ts
index 78fdfeb02..2f082e6d0 100644
--- a/packages/core/src/markdown/html-to-mdast.ts
+++ b/packages/core/src/markdown/html-to-mdast.ts
@@ -1,3 +1,4 @@
+
import type { Root as HastRoot } from 'hast';
import type { Root as MdastRoot } from 'mdast';
import rehypeParse from 'rehype-parse';
@@ -75,7 +76,8 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRo
throw new HtmlPayloadTooLargeError(html.length, maxBytes);
}
- const processor = unified().use(rehypeParse, { fragment: true });
+ const processor = unified()
+ .use(rehypeParse, { fragment: true });
for (const plugin of cleanupPlugins) {
processor.use(plugin);
diff --git a/packages/core/src/markdown/image-promoter.ts b/packages/core/src/markdown/image-promoter.ts
index 706fa9afb..994eb6917 100644
--- a/packages/core/src/markdown/image-promoter.ts
+++ b/packages/core/src/markdown/image-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Image, Paragraph, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts
index 58a810193..064fa0773 100644
--- a/packages/core/src/markdown/index.ts
+++ b/packages/core/src/markdown/index.ts
@@ -1,3 +1,4 @@
+
import {
type FromProseMirrorOptions,
fromPmMark,
@@ -164,6 +165,7 @@ export class MarkdownManager {
}
}
+
const registry = createRegistry();
function destructureAttrs(
@@ -212,6 +214,7 @@ function destructureAttrs(
return result;
}
+
function hasDirtyDescendant(node: PmNode): boolean {
let found = false;
node.descendants((child) => {
@@ -230,6 +233,7 @@ function effectiveDirty(node: PmNode): boolean {
return node.attrs.sourceDirty || hasDirtyDescendant(node);
}
+
function isEmptyMdastParagraph(node: MdastNodes): boolean {
if (node.type !== 'paragraph') return false;
const children = node.children ?? [];
@@ -298,6 +302,7 @@ function extractTextFromMdastNodes(nodes: MdastNodes[]): string {
return out;
}
+
import {
AUDIO_EXTENSIONS,
FILE_ATTACHMENT_EXTENSIONS,
@@ -547,6 +552,7 @@ function buildMdastToPmHandlers(
}));
}
+
if (m.emphasis) {
handlers.emphasis = toPmMark(m.emphasis, (node: Emphasis) => ({
sourceDelimiter: node.data?.sourceDelimiter ?? '*',
@@ -626,6 +632,7 @@ function buildMdastToPmHandlers(
}));
}
+
if (m.link) {
const sourceLiteralMark = m.sourceLiteral;
handlers.link = (node: Link, _parent: MdastParent, state: MdastToPmState) => {
@@ -904,6 +911,7 @@ function buildMdastToPmHandlers(
};
}
+
const blockUnknownHandler = (node: {
type: string;
position?: { start: { offset: number }; end: { offset: number } };
@@ -1029,6 +1037,7 @@ function buildMdastToPmHandlers(
return handlers as RemarkProseMirrorOptions['handlers'];
}
+
function buildPmToMdastHandlers(schema: Schema): {
nodeHandlers: FromProseMirrorOptions<string, string>['nodeHandlers'];
markHandlers: FromProseMirrorOptions<string, string>['markHandlers'];
diff --git a/packages/core/src/markdown/math-promoter.ts b/packages/core/src/markdown/math-promoter.ts
index a5b511f25..bd79424c7 100644
--- a/packages/core/src/markdown/math-promoter.ts
+++ b/packages/core/src/markdown/math-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Code, Root } from 'mdast';
import type { Math as MdastMath } from 'mdast-util-math';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.ts
index 679c72ef7..f84967798 100644
--- a/packages/core/src/markdown/mdast-to-hast-handlers.ts
+++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts
@@ -1,3 +1,4 @@
+
import type { Comment, Element, ElementContent, Properties } from 'hast';
import type { FootnoteDefinition, FootnoteReference } from 'mdast';
import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';
diff --git a/packages/core/src/markdown/mdast-to-html.ts b/packages/core/src/markdown/mdast-to-html.ts
index 79072375c..153fdc1d8 100644
--- a/packages/core/src/markdown/mdast-to-html.ts
+++ b/packages/core/src/markdown/mdast-to-html.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root as HastRoot } from 'hast';
import type { Root as MdastRoot } from 'mdast';
import { normalizeUri } from 'micromark-util-sanitize-uri';
diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts
index 35d74b85e..a58f8c720 100644
--- a/packages/core/src/markdown/merged-walker.ts
+++ b/packages/core/src/markdown/merged-walker.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Parent, Root } from 'mdast';
import { SKIP, visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/mermaid-promoter.ts b/packages/core/src/markdown/mermaid-promoter.ts
index 1a897fca8..17ed2b829 100644
--- a/packages/core/src/markdown/mermaid-promoter.ts
+++ b/packages/core/src/markdown/mermaid-promoter.ts
@@ -1,3 +1,4 @@
+
import type { Code, Root } from 'mdast';
import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import { visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.ts
index 9864b3b60..7e4a9a48b 100644
--- a/packages/core/src/markdown/parse-with-fallback.ts
+++ b/packages/core/src/markdown/parse-with-fallback.ts
@@ -168,6 +168,7 @@ export function parseRecursive(
}
}
+
interface VFilePlace {
offset?: number;
start?: { offset?: number };
@@ -186,6 +187,7 @@ function extractErrorOffset(err: unknown): number | undefined {
return undefined;
}
+
interface Region {
start: number;
end: number;
@@ -209,6 +211,7 @@ function nearestBlankLineAfter(src: string, offset: number): number | null {
return null;
}
+
export interface TagEvent {
kind: 'open' | 'close' | 'self-close';
name: string;
@@ -355,6 +358,7 @@ function findFallbackRegion(src: string, errorOffset: number): Region {
return { start: blockStart, end: blockEnd };
}
+
interface SourceBlock {
src: string;
start: number;
@@ -452,6 +456,7 @@ function tryPerBlockFallback(
};
}
+
function wholeDocRawText(source: string): JSONContent {
return {
type: 'doc',
diff --git a/packages/core/src/markdown/parser-drop-closure.ts b/packages/core/src/markdown/parser-drop-closure.ts
index e6ab4fd57..43d420187 100644
--- a/packages/core/src/markdown/parser-drop-closure.ts
+++ b/packages/core/src/markdown/parser-drop-closure.ts
@@ -1,3 +1,4 @@
+
export type DroppedTokenAdjudication =
| {
kind: 'format-dof-axis';
diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts
index 5d6db7aa8..5b9550e90 100644
--- a/packages/core/src/markdown/pipeline.ts
+++ b/packages/core/src/markdown/pipeline.ts
@@ -1,3 +1,4 @@
+
import {
type FromProseMirrorOptions,
fromProseMirror,
diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.ts
index 248685cfe..4ac5731e7 100644
--- a/packages/core/src/markdown/position-aware-join.ts
+++ b/packages/core/src/markdown/position-aware-join.ts
@@ -1,3 +1,4 @@
+
import type { Join } from 'mdast-util-to-markdown';
import type { Position } from 'unist';
diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts
index 50a193a16..240be90e6 100644
--- a/packages/core/src/markdown/position-slice.ts
+++ b/packages/core/src/markdown/position-slice.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Root } from 'mdast';
import { visit } from 'unist-util-visit';
import type { VFile } from 'vfile';
@@ -466,7 +467,8 @@ export function applyPositionSliceToNode(
const first = source[startOff];
if (first !== '[' && first !== '<' && !node.title) {
node.data.sourceStyle = 'gfm-autolink';
- } else if (first === '[') {
+ }
+ else if (first === '[') {
const slice = source.slice(startOff, endOff);
const closeBracketIdx = slice.lastIndexOf('](');
if (closeBracketIdx !== -1) {
diff --git a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
index db10fa54b..ca225ce9b 100644
--- a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
+++ b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts
@@ -1,3 +1,4 @@
+
import type { Comment, Element, Root, Text } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
index 34068d103..3a2524a9a 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts
@@ -1,3 +1,4 @@
+
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
index 93151d43b..2279a0189 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
index 188f02d83..a0f467c42 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
index 4ba2c3418..4e8542a21 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
index 1a74b1272..a3c96b24a 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts
@@ -1,3 +1,4 @@
+
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
index de35e7b79..e95cf43d6 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts
@@ -1,3 +1,4 @@
+
import type { Comment, Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
index b4be1b2ea..ffdb4d5b5 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts
@@ -1,3 +1,4 @@
+
import type { Element, ElementContent, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
index 3c8d88b7c..e2a96d1a7 100644
--- a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
+++ b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts
@@ -1,3 +1,4 @@
+
import type { Element, Root } from 'hast';
import type { Plugin } from 'unified';
diff --git a/packages/core/src/markdown/resolve-image-url.ts b/packages/core/src/markdown/resolve-image-url.ts
index cf2c9da8e..0389575ac 100644
--- a/packages/core/src/markdown/resolve-image-url.ts
+++ b/packages/core/src/markdown/resolve-image-url.ts
@@ -1,3 +1,4 @@
+
import { isRelativeUrl } from './safe-url.ts';
function isDevDiagnosticContext(): boolean {
diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts
index 4d3250d3a..8be632ba4 100644
--- a/packages/core/src/markdown/safe-url.ts
+++ b/packages/core/src/markdown/safe-url.ts
@@ -1,3 +1,4 @@
+
export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;
const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');
diff --git a/packages/core/src/markdown/serialize-helpers.ts b/packages/core/src/markdown/serialize-helpers.ts
index 46cc6a074..542204403 100644
--- a/packages/core/src/markdown/serialize-helpers.ts
+++ b/packages/core/src/markdown/serialize-helpers.ts
@@ -1,3 +1,4 @@
+
import type { Node as PmNode } from '@tiptap/pm/model';
import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import type { PropDef, SerializeContext } from '../registry/types.ts';
diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.ts
index be97d4d62..fe73659e6 100644
--- a/packages/core/src/markdown/single-dollar-math-promoter.ts
+++ b/packages/core/src/markdown/single-dollar-math-promoter.ts
@@ -1,3 +1,4 @@
+
import type { PhrasingContent, Root, Text } from 'mdast';
import type { InlineMath } from 'mdast-util-math';
import { SKIP, visit } from 'unist-util-visit';
diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts
index 1a2fb5d50..e1161455b 100644
--- a/packages/core/src/markdown/to-markdown-handlers.ts
+++ b/packages/core/src/markdown/to-markdown-handlers.ts
@@ -1,3 +1,4 @@
+
import type { Nodes, Parents } from 'mdast';
import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx';
import type { Handle, Info, State } from 'mdast-util-to-markdown';
diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.ts
index cd245c38c..c5bfd2386 100644
--- a/packages/core/src/markdown/unknown-mdast-guard.ts
+++ b/packages/core/src/markdown/unknown-mdast-guard.ts
@@ -1,3 +1,4 @@
+
import type { Root as MdastRoot } from 'mdast';
import type { VFile } from 'vfile';
diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.ts
index a100cad76..1b75ba564 100644
--- a/packages/core/src/markdown/whitespace-char-ref.ts
+++ b/packages/core/src/markdown/whitespace-char-ref.ts
@@ -1,3 +1,4 @@
+
const INLINE_WHITESPACE_BY_CODE: ReadonlyMap<number, string> = new Map([
[0x20, ' '],
[0x09, '\t'],
diff --git a/packages/core/src/markdown/wiki-link-micromark.ts b/packages/core/src/markdown/wiki-link-micromark.ts
index 6dad26200..afd9936fe 100644
--- a/packages/core/src/markdown/wiki-link-micromark.ts
+++ b/packages/core/src/markdown/wiki-link-micromark.ts
@@ -17,6 +17,7 @@ declare module 'micromark-util-types' {
}
}
+
const CODE_BANG = 33; // !
const CODE_LBRACKET = 91; // [
const CODE_RBRACKET = 93; // ]
@@ -260,6 +261,7 @@ export function wikiLinkSyntax(): Extension {
};
}
+
function enterWikiLink(this: CompileContext, token: Token) {
this.enter(
{
@@ -385,6 +387,7 @@ export const wikiLinkToMarkdown: {
unsafe: [{ character: '[', inConstruct: ['phrasing'] }],
};
+
const MICROMARK_EXT = wikiLinkSyntax();
export function remarkWikiLink(this: Processor) {
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index 84809463e..8dc135b65 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -15525,7 +15525,15 @@ export function createApiExtension(options: ApiExtensionOptions): Extension {
const end = Math.min(content.length, index + normalizedQuery.length + 120);
const prefix = start > 0 ? '…' : '';
const suffix = end < content.length ? '…' : '';
- return `${prefix}${content.slice(start, end).replace(/\s+/g, ' ').trim()}${suffix}`;
+ // slice() cuts on UTF-16 code units, so a boundary landing mid-emoji leaves a
+ // lone surrogate. Replace any unpaired surrogate with U+FFFD so strict JSON-RPC
+ // clients (Rust / pydantic parsers) don't reject the response as invalid UTF-8.
+ // (String.toWellFormed() would do this but needs the es2024 lib in every consumer.)
+ const snippet = `${prefix}${content.slice(start, end).replace(/\s+/g, ' ').trim()}${suffix}`;
+ return snippet.replace(
+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
+ '\uFFFD',
+ );
}
function parseSearchIntent(value: unknown): WorkspaceSearchIntent {
diff --git a/packages/server/src/api-search.test.ts b/packages/server/src/api-search.test.ts
index 31f480eb3..6a6fbf1d8 100644
--- a/packages/server/src/api-search.test.ts
+++ b/packages/server/src/api-search.test.ts
@@ -171,6 +171,53 @@ describe('GET /api/search', () => {
}
});
+ test('snippet truncated mid-emoji carries no lone surrogate', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-search-'));
+ try {
+ // buildSearchSnippet cuts the snippet at `matchIndex + query.length + 120`
+ // code units. The filler run is sized so that boundary falls between the two
+ // UTF-16 halves of the emoji, which is what leaves a lone surrogate.
+ const content = `# E\n\ncrdt${'a'.repeat(119)}😀 tail text follows here\n`;
+ writeFileSync(join(dir, 'emoji.md'), content, 'utf-8');
+
+ const result = await callSearch(dir, '/api/search?query=crdt&intent=full_text');
+ expect(result.status).toBe(200);
+ const body = JSON.parse(result.body) as { results?: Array<{ snippet?: string }> };
+ const snippet = body.results?.[0]?.snippet ?? '';
+ expect(snippet.length).toBeGreaterThan(0);
+ // A lone surrogate is not encodable as UTF-8, so a round-trip through the
+ // wire encoding (what stdio serialization does) replaces it and diverges.
+ // Well-formed snippets round-trip unchanged. This is the failure strict
+ // JSON-RPC clients hit, without coupling the test to the fix's mechanism.
+ expect(Buffer.from(snippet, 'utf8').toString('utf8')).toBe(snippet);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test('snippet truncated at the start boundary carries no lone low surrogate', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-search-'));
+ try {
+ // The other boundary: buildSearchSnippet also cuts at `matchIndex - 80`,
+ // which can split an emoji and leave a lone LOW surrogate at the snippet
+ // start (the regex's second arm). Layout: `# E\n\n` (5) + emoji (2) + 78
+ // filler + a space puts the emoji halves across code units 5/6 and the
+ // `crdt` token at index 86, so `start = 86 - 80 = 6` lands on the emoji low
+ // half. crdt must be its own token, else the token-prefix search finds no match.
+ const content = `# E\n\n😀${'a'.repeat(78)} crdt tail text follows here to pad it out\n`;
+ writeFileSync(join(dir, 'emoji-start.md'), content, 'utf-8');
+
+ const result = await callSearch(dir, '/api/search?query=crdt&intent=full_text');
+ expect(result.status).toBe(200);
+ const body = JSON.parse(result.body) as { results?: Array<{ snippet?: string }> };
+ const snippet = body.results?.[0]?.snippet ?? '';
+ expect(snippet.length).toBeGreaterThan(0);
+ expect(Buffer.from(snippet, 'utf8').toString('utf8')).toBe(snippet);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
test('supports POST bodies for shared search clients', async () => {
const dir = mkdtempSync(join(tmpdir(), 'ok-search-'));
try {