Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
feat(text-viewer): open code-language files in the sidebar with syntax highlight (#1893)
Abraham946be811c77e
diff --git a/packages/app/package.json b/packages/app/package.jsonindex b6c4fd0ef..899666d3a 100644--- a/packages/app/package.json+++ b/packages/app/package.json@@ -50,7 +50,7 @@{"name": "all JS chunks combined (gzipped)","path": "dist/assets/*.js",- "limit": "2.75 MB",+ "limit": "2.85 MB","gzip": true,"running": false},diff --git a/packages/app/src/components/TextViewer.tsx b/packages/app/src/components/TextViewer.tsxindex d3abbe480..7415b644b 100644--- a/packages/app/src/components/TextViewer.tsx+++ b/packages/app/src/components/TextViewer.tsx@@ -1,13 +1,12 @@-import { json } from '@codemirror/lang-json';-import { type Language, StreamLanguage } from '@codemirror/language';-import { toml } from '@codemirror/legacy-modes/mode/toml';import { EditorState } from '@codemirror/state';import { EditorView } from '@codemirror/view';+import { codeLanguageForExtension } from '@inkeep/open-knowledge-core';import { Trans, useLingui } from '@lingui/react/macro';import { basicDarkInit, basicLightInit } from '@uiw/codemirror-theme-basic';import { basicSetup } from 'codemirror';import { useTheme } from 'next-themes';import { useEffect, useRef, useState } from 'react';+import { loadCodeMirrorLanguageForExtension } from './text-viewer-languages';const darkTheme = basicDarkInit({settings: {@@ -23,18 +22,6 @@ const lightTheme = basicLightInit({},});-function pickLanguageExtension(extension: string): Language | null {- switch (extension.toLowerCase()) {- case 'json':- case 'canvas':- return json().language;- case 'toml':- return StreamLanguage.define(toml);- default:- return null;- }-}-interface TextViewerProps {src: string;fileName: string;@@ -101,25 +88,32 @@ export function TextViewer({ src, fileName, extension }: TextViewerProps) {if (!containerRef.current) return;if (loadedContent === null) return;- const language = pickLanguageExtension(extension);+ const normalized = extension.toLowerCase();+ const canonical = codeLanguageForExtension(normalized);const theme = resolvedTheme === 'dark' ? darkTheme : lightTheme;- const extensions = [- basicSetup,- ...(language ? [language] : []),- EditorView.editable.of(true),- EditorState.readOnly.of(true),- EditorView.lineWrapping,- theme,- ];-- const view = new EditorView({- state: EditorState.create({ doc: loadedContent, extensions }),- parent: containerRef.current,+ let aborted = false;+ let view: EditorView | null = null;+ void loadCodeMirrorLanguageForExtension(normalized, canonical).then((language) => {+ if (aborted) return;+ if (!containerRef.current) return;+ const extensions = [+ basicSetup,+ ...(language ? [language] : []),+ EditorView.editable.of(true),+ EditorState.readOnly.of(true),+ EditorView.lineWrapping,+ theme,+ ];+ view = new EditorView({+ state: EditorState.create({ doc: loadedContent, extensions }),+ parent: containerRef.current,+ });+ viewRef.current = view;});- viewRef.current = view;return () => {- view.destroy();+ aborted = true;+ view?.destroy();viewRef.current = null;};}, [loadedContent, extension, resolvedTheme]);diff --git a/packages/app/src/components/text-viewer-languages.ts b/packages/app/src/components/text-viewer-languages.tsnew file mode 100644index 000000000..da5e3ea14--- /dev/null+++ b/packages/app/src/components/text-viewer-languages.ts@@ -0,0 +1,123 @@+import type { Language } from '@codemirror/language';++const cache = new Map<string, Promise<Language | null>>();++function loadCodeMirrorLanguage(canonical: string): Promise<Language | null> {+ const cached = cache.get(canonical);+ if (cached) return cached;+ const promise = resolveLanguage(canonical);+ cache.set(canonical, promise);+ return promise;+}++async function resolveLanguage(canonical: string): Promise<Language | null> {+ switch (canonical) {+ case 'bash':+ case 'shell':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/shell')).shell,+ );+ case 'c':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/clike')).c,+ );+ case 'cpp':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/clike')).cpp,+ );+ case 'csharp':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/clike')).csharp,+ );+ case 'css':+ case 'less':+ case 'scss':+ return (await import('@codemirror/lang-css')).css().language;+ case 'diff':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/diff')).diff,+ );+ case 'go':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/go')).go,+ );+ case 'ini':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/properties')).properties,+ );+ case 'java':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/clike')).java,+ );+ case 'javascript':+ return (await import('@codemirror/lang-javascript')).javascript().language;+ case 'json':+ return (await import('@codemirror/lang-json')).json().language;+ case 'kotlin':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/clike')).kotlin,+ );+ case 'lua':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/lua')).lua,+ );+ case 'markdown':+ return (await import('@codemirror/lang-markdown')).markdown().language;+ case 'objectivec':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/clike')).objectiveC,+ );+ case 'perl':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/perl')).perl,+ );+ case 'python':+ return (await import('@codemirror/lang-python')).python().language;+ case 'r':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/r')).r,+ );+ case 'ruby':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/ruby')).ruby,+ );+ case 'rust':+ return (await import('@codemirror/lang-rust')).rust().language;+ case 'sql':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/sql')).standardSQL,+ );+ case 'swift':+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/swift')).swift,+ );+ case 'typescript':+ return (await import('@codemirror/lang-javascript')).javascript({+ typescript: true,+ jsx: true,+ }).language;+ case 'xml':+ return (await import('@codemirror/lang-html')).html().language;+ case 'yaml':+ return (await import('@codemirror/lang-yaml')).yaml().language;+ default:+ return null;+ }+}++export async function loadCodeMirrorLanguageForExtension(+ extension: string,+ canonical: string | null,+): Promise<Language | null> {+ if (extension === 'canvas') return (await import('@codemirror/lang-json')).json().language;+ if (canonical) {+ const lang = await loadCodeMirrorLanguage(canonical);+ if (lang) return lang;+ }+ if (extension === 'toml') {+ return (await import('@codemirror/language')).StreamLanguage.define(+ (await import('@codemirror/legacy-modes/mode/toml')).toml,+ );+ }+ return null;+}diff --git a/packages/app/tests/integration/sidebar-show-all-files.test.ts b/packages/app/tests/integration/sidebar-show-all-files.test.tsindex 00852bd76..b4042bdf9 100644--- a/packages/app/tests/integration/sidebar-show-all-files.test.ts+++ b/packages/app/tests/integration/sidebar-show-all-files.test.ts@@ -106,7 +106,7 @@ describe('/api/documents?showAll=true', () => {const indexTs = body.documents.find((e) => e.kind === 'asset' && e.path === 'src/index.ts');expect(indexTs).toBeTruthy();expect(indexTs?.assetExt).toBe('ts');- expect(indexTs?.mediaKind).toBe(null);+ expect(indexTs?.mediaKind).toBe('text');const analysisPy = body.documents.find((e) => e.kind === 'asset' && e.path === 'analysis.py');expect(analysisPy).toBeTruthy();diff --git a/packages/core/src/constants/code-languages.test.ts b/packages/core/src/constants/code-languages.test.tsnew file mode 100644index 000000000..bb01a658e--- /dev/null+++ b/packages/core/src/constants/code-languages.test.ts@@ -0,0 +1,138 @@+import { describe, expect, test } from 'bun:test';+import {+ CODE_FILE_BARE_NAMES_TO_LANGUAGE,+ CODE_FILE_EXTENSIONS,+ CODE_FILE_EXTENSIONS_TO_LANGUAGE,+ codeLanguageForBareFilename,+ codeLanguageForExtension,+} from './code-languages';++describe('codeLanguageForExtension', () => {+ test('maps the canonical extensions to their codeblock language IDs', () => {+ expect(codeLanguageForExtension('ts')).toBe('typescript');+ expect(codeLanguageForExtension('tsx')).toBe('typescript');+ expect(codeLanguageForExtension('js')).toBe('javascript');+ expect(codeLanguageForExtension('py')).toBe('python');+ expect(codeLanguageForExtension('rs')).toBe('rust');+ expect(codeLanguageForExtension('go')).toBe('go');+ expect(codeLanguageForExtension('rb')).toBe('ruby');+ expect(codeLanguageForExtension('sh')).toBe('bash');+ expect(codeLanguageForExtension('cpp')).toBe('cpp');+ expect(codeLanguageForExtension('h')).toBe('c');+ expect(codeLanguageForExtension('hpp')).toBe('cpp');+ expect(codeLanguageForExtension('yaml')).toBe('yaml');+ expect(codeLanguageForExtension('xml')).toBe('xml');+ expect(codeLanguageForExtension('html')).toBeNull();+ expect(codeLanguageForExtension('htm')).toBeNull();+ expect(codeLanguageForExtension('svg')).toBeNull();+ });++ test('strips a leading dot from the extension input', () => {+ expect(codeLanguageForExtension('.ts')).toBe('typescript');+ expect(codeLanguageForExtension('.py')).toBe('python');+ });++ test('is case-insensitive', () => {+ expect(codeLanguageForExtension('TS')).toBe('typescript');+ expect(codeLanguageForExtension('PY')).toBe('python');+ expect(codeLanguageForExtension('Cpp')).toBe('cpp');+ });++ test('returns null for an unknown extension', () => {+ expect(codeLanguageForExtension('txt')).toBeNull();+ expect(codeLanguageForExtension('docx')).toBeNull();+ expect(codeLanguageForExtension('zip')).toBeNull();+ expect(codeLanguageForExtension('')).toBeNull();+ });++ test('aliases consolidate to the same canonical ID', () => {+ const javascriptAliases = ['js', 'jsx', 'mjs', 'cjs'];+ for (const alias of javascriptAliases) {+ expect(codeLanguageForExtension(alias)).toBe('javascript');+ }++ const typescriptAliases = ['ts', 'tsx'];+ for (const alias of typescriptAliases) {+ expect(codeLanguageForExtension(alias)).toBe('typescript');+ }++ const bashAliases = ['sh', 'zsh', 'bash'];+ for (const alias of bashAliases) {+ expect(codeLanguageForExtension(alias)).toBe('bash');+ }+ });+});++describe('codeLanguageForBareFilename', () => {+ test('Makefile (any case) resolves to makefile', () => {+ expect(codeLanguageForBareFilename('Makefile')).toBe('makefile');+ expect(codeLanguageForBareFilename('makefile')).toBe('makefile');+ expect(codeLanguageForBareFilename('MAKEFILE')).toBe('makefile');+ });++ test('Dockerfile resolves to bash (shell-shaped commands)', () => {+ expect(codeLanguageForBareFilename('Dockerfile')).toBe('bash');+ expect(codeLanguageForBareFilename('dockerfile')).toBe('bash');+ });++ test('Gemfile / Rakefile resolve to ruby', () => {+ expect(codeLanguageForBareFilename('Gemfile')).toBe('ruby');+ expect(codeLanguageForBareFilename('Rakefile')).toBe('ruby');+ });++ test('returns null for an unrecognized bare filename', () => {+ expect(codeLanguageForBareFilename('README')).toBeNull();+ expect(codeLanguageForBareFilename('LICENSE')).toBeNull();+ });+});++describe('CODE_FILE_EXTENSIONS — internal consistency', () => {+ test('every extension key resolves through codeLanguageForExtension', () => {+ for (const ext of CODE_FILE_EXTENSIONS) {+ expect(codeLanguageForExtension(ext)).not.toBeNull();+ }+ });++ test('every canonical language ID is present in CODE_BLOCK_LANGUAGES', () => {+ const knownCanonical = new Set([+ 'plaintext',+ 'bash',+ 'c',+ 'cpp',+ 'csharp',+ 'css',+ 'diff',+ 'go',+ 'graphql',+ 'ini',+ 'java',+ 'javascript',+ 'json',+ 'kotlin',+ 'less',+ 'lua',+ 'makefile',+ 'markdown',+ 'objectivec',+ 'perl',+ 'php',+ 'python',+ 'r',+ 'ruby',+ 'rust',+ 'scss',+ 'shell',+ 'sql',+ 'swift',+ 'typescript',+ 'xml',+ 'yaml',+ ]);+ for (const lang of Object.values(CODE_FILE_EXTENSIONS_TO_LANGUAGE)) {+ expect(knownCanonical.has(lang)).toBe(true);+ }+ for (const lang of Object.values(CODE_FILE_BARE_NAMES_TO_LANGUAGE)) {+ expect(knownCanonical.has(lang)).toBe(true);+ }+ });+});diff --git a/packages/core/src/constants/code-languages.ts b/packages/core/src/constants/code-languages.tsnew file mode 100644index 000000000..ff19585e4--- /dev/null+++ b/packages/core/src/constants/code-languages.ts@@ -0,0 +1,99 @@+export const CODE_FILE_EXTENSIONS_TO_LANGUAGE: Readonly<Record<string, string>> = {+ sh: 'bash',+ zsh: 'bash',+ bash: 'bash',++ c: 'c',+ cc: 'cpp',+ cpp: 'cpp',+ cxx: 'cpp',+ 'c++': 'cpp',+ h: 'c',+ hh: 'cpp',+ hpp: 'cpp',++ cs: 'csharp',+ css: 'css',+ less: 'less',+ scss: 'scss',+ sass: 'scss',++ diff: 'diff',+ patch: 'diff',++ go: 'go',++ gql: 'graphql',+ graphql: 'graphql',++ ini: 'ini',+ cfg: 'ini',+ conf: 'ini',+ properties: 'ini',++ java: 'java',++ js: 'javascript',+ jsx: 'javascript',+ mjs: 'javascript',+ cjs: 'javascript',++ jsonc: 'json',++ kt: 'kotlin',+ kts: 'kotlin',++ lua: 'lua',+ makefile: 'makefile',++ md: 'markdown',+ mdx: 'markdown',++ m: 'objectivec',+ mm: 'objectivec',++ pl: 'perl',+ pm: 'perl',++ php: 'php',+ phtml: 'php',++ py: 'python',+ pyi: 'python',+ pyx: 'python',++ r: 'r',+ rb: 'ruby',+ rs: 'rust',++ sql: 'sql',+ swift: 'swift',++ ts: 'typescript',+ tsx: 'typescript',++ xml: 'xml',++ yaml: 'yaml',+ yml: 'yaml',+};++export const CODE_FILE_BARE_NAMES_TO_LANGUAGE: Readonly<Record<string, string>> = {+ makefile: 'makefile',+ dockerfile: 'bash',+ gemfile: 'ruby',+ rakefile: 'ruby',+};++export const CODE_FILE_EXTENSIONS: ReadonlySet<string> = new Set(+ Object.keys(CODE_FILE_EXTENSIONS_TO_LANGUAGE),+);++export function codeLanguageForExtension(ext: string): string | null {+ const normalized = ext.toLowerCase().replace(/^\./, '');+ return CODE_FILE_EXTENSIONS_TO_LANGUAGE[normalized] ?? null;+}++export function codeLanguageForBareFilename(name: string): string | null {+ return CODE_FILE_BARE_NAMES_TO_LANGUAGE[name.toLowerCase()] ?? null;+}diff --git a/packages/core/src/constants/upload.ts b/packages/core/src/constants/upload.tsindex dce9b1072..ec480f151 100644--- a/packages/core/src/constants/upload.ts+++ b/packages/core/src/constants/upload.ts@@ -343,6 +343,10 @@ assertSubset(export const TEXT_VIEWER_FALLBACK_EXTENSIONS: ReadonlySet<string> = new Set(['base', 'canvas']);+import { CODE_FILE_EXTENSIONS } from './code-languages';++export { CODE_FILE_EXTENSIONS };+export const LINKABLE_ASSET_EXTENSIONS: ReadonlySet<string> = new Set([...ASSET_EXTENSIONS,...TEXT_VIEWER_FALLBACK_EXTENSIONS,@@ -356,5 +360,6 @@ export function mediaKindForSidebarAssetExtension(ext: string): InlineAssetMediaif (SIDEBAR_PDF_ASSET_EXTENSIONS.has(normalized)) return 'pdf';if (SIDEBAR_TEXT_ASSET_EXTENSIONS.has(normalized)) return 'text';if (TEXT_VIEWER_FALLBACK_EXTENSIONS.has(normalized)) return 'text';+ if (CODE_FILE_EXTENSIONS.has(normalized)) return 'text';return null;}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex a27f34e51..6f098aae1 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -174,6 +174,12 @@ export {type ConfigDocName,SYSTEM_DOC_NAME,} from './constants/cc1.ts';+export {+ CODE_FILE_BARE_NAMES_TO_LANGUAGE,+ CODE_FILE_EXTENSIONS_TO_LANGUAGE,+ codeLanguageForBareFilename,+ codeLanguageForExtension,+} from './constants/code-languages.ts';export type { CreateNewBannerKind } from './constants/create-new-banner.ts';export {CREATE_NEW_PROJECT_FAILURE_REASONS,@@ -223,6 +229,7 @@ export {ALLOWED_VIDEO_MIME_TYPES,ASSET_EXTENSIONS,AUDIO_EXTENSIONS,+ CODE_FILE_EXTENSIONS,DEFAULT_ATTACHMENT_FOLDER_PATH,DEFAULT_DEDUP_MODE,DEFAULT_DEDUP_UI,