Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
/**
* The import/export half of edit-docs, with no Electron dependency: `edit-docs.ts` runs it
* behind a window, `sync-docs.ts` runs it headless.
*/
import { BackupService, initializeCore, type MessagingProvider, type NoteMeta, type NoteMetaFile } from "@triliumnext/core";
import AsyncLocalStorageExecutionContext from "@triliumnext/server/src/cls_provider.js";
import NodejsCryptoProvider from "@triliumnext/server/src/crypto_provider.js";
import ServerPlatformProvider from "@triliumnext/server/src/platform_provider.js";
import { serverImageProvider } from "@triliumnext/server/src/services/image_provider.js";
import BetterSqlite3Provider from "@triliumnext/server/src/sql_provider.js";
import NodejsZipProvider from "@triliumnext/server/src/zip_provider.js";
import { type Archiver, ZipArchive } from "archiver";
import { createWriteStream, readFileSync, type WriteStream } from "fs";
import fs from "fs/promises";
import { load } from "js-yaml";
import path from "path";
import { deferred } from "../../../packages/commons/src/index.js";
import { buildHelpBundle } from "./help_bundle_generator.js";
import { buildHelpMeta, serializeHelpMeta, stripAppVersion } from "./help_meta_generator.js";
import { openApiToMarkdown, type OpenApiSpec } from "./openapi_markdown.js";
import { runTypeDoc, typeDocToDeclarations, type TypeDocMarkdownOptions, typeDocToMarkdown } from "./typedoc_markdown.js";
export interface NoteMapping {
rootNoteId: string;
path: string;
format: "markdown";
ignoredFiles?: string[];
exportOnly?: boolean;
/**
* Where to write the in-app help tree derived from this mapping's export, if it is the one
* backing the User Guide. Relative to the configuration file.
*/
helpMeta?: string;
/** Where to write the rendered content of that tree. Relative to the configuration file. */
helpContent?: string;
/** Pages whose Markdown is generated from an OpenAPI document before every import. */
openApiPages?: OpenApiPage[];
/** Pages whose Markdown is generated from TypeScript declarations before every import. */
typeDocPages?: TypeDocPage[];
}
export interface OpenApiPage {
/** A text note of this mapping with a Markdown data file. */
noteId: string;
/** The OpenAPI document in YAML. Relative to the configuration file. */
spec: string;
}
export interface TypeDocPage extends TypeDocMarkdownOptions {
/** A text note of this mapping with a Markdown data file. */
noteId: string;
/** The module whose exports the page documents. Relative to the configuration file. */
entryPoint: string;
/** The tsconfig that compiles it. Relative to the configuration file. */
tsconfig: string;
/**
* Where to also write the declarations as compact TypeScript, for the LLM skill sheets that
* `load_skill` serves. Relative to the configuration file.
*/
skillReference?: string;
}
export interface Config {
baseUrl: string;
noteMappings: NoteMapping[];
}
/**
* Reads `edit-docs-config.yaml`: the given path, else the current directory, else the repository
* root. Mapping paths resolve against the config file's directory.
*/
export async function loadConfig(configPath?: string): Promise<Config> {
let resolvedPath = configPath
? path.resolve(configPath)
: path.join(process.cwd(), "edit-docs-config.yaml");
const exists = await fs.access(resolvedPath).then(() => true).catch(() => false);
if (!exists && !configPath) {
resolvedPath = path.join(__dirname, "../../../edit-docs-config.yaml");
}
const config = load(await fs.readFile(resolvedPath, "utf-8")) as Config;
const configDir = path.dirname(resolvedPath);
return {
baseUrl: config.baseUrl,
noteMappings: config.noteMappings.map((mapping) => ({
...mapping,
path: path.resolve(configDir, mapping.path),
helpMeta: mapping.helpMeta ? path.resolve(configDir, mapping.helpMeta) : undefined,
helpContent: mapping.helpContent ? path.resolve(configDir, mapping.helpContent) : undefined,
openApiPages: mapping.openApiPages?.map((page) => ({ ...page, spec: path.resolve(configDir, page.spec) })),
typeDocPages: mapping.typeDocPages?.map((page) => ({
...page,
entryPoint: path.resolve(configDir, page.entryPoint),
tsconfig: path.resolve(configDir, page.tsconfig),
skillReference: page.skillReference ? path.resolve(configDir, page.skillReference) : undefined
}))
}))
};
}
// Stub backup service (not used in edit-docs, but required by initializeCore)
class StubBackupService extends BackupService {
constructor() {
super({ getOption: () => "", getOptionOrNull: () => null, getOptionBool: () => false, setOption: () => {} });
}
scheduleBackups(): void {}
async backupNow(_name: string): Promise<string> {
throw new Error("Backup not supported in edit-docs");
}
async getExistingBackups() {
return [];
}
async getBackupContent(_filePath: string): Promise<Uint8Array | null> {
return null;
}
}
/**
* Initializes core against an in-memory database. `messaging` is the IPC provider when a window
* is attached; headless runs pass none.
*/
export async function initializeDocsCore(messaging?: MessagingProvider) {
const dbProvider = new BetterSqlite3Provider();
dbProvider.loadFromMemory();
const { serverZipExportProviderFactory } = await import("@triliumnext/server/src/services/export/zip/factory.js");
await initializeCore({
dbConfig: {
provider: dbProvider,
isReadOnly: false,
async onTransactionCommit() {
if (!messaging) {
return;
}
const { ws } = await import("@triliumnext/core");
ws.sendTransactionEntityChangesToAllClients();
},
onTransactionRollback: () => {}
},
crypto: new NodejsCryptoProvider(),
zip: new NodejsZipProvider(),
zipExportProviderFactory: serverZipExportProviderFactory,
executionContext: new AsyncLocalStorageExecutionContext(),
platform: new ServerPlatformProvider(),
schema: readFileSync(require.resolve("@triliumnext/core/src/assets/schema.sql"), "utf-8"),
translations: (await import("@triliumnext/server/src/services/i18n.js")).initializeTranslationsWithParams,
messaging,
getDemoArchive: async () => null,
backup: new StubBackupService(),
image: serverImageProvider
});
}
export async function importData(path: string) {
const buffer = await createImportZip(path);
const { zipImportService, TaskContext, becca } = (await import("@triliumnext/core"));
const context = new TaskContext("no-progress-reporting", "importNotes", null);
const rootNote = becca.getRoot();
if (!rootNote) {
throw new Error("Missing root note for import.");
}
await zipImportService.importZip(context, buffer, rootNote, {
preserveIds: true
});
}
/** Imports every mapping that is not `exportOnly`, in config order. */
export async function importMappings(mappings: NoteMapping[]) {
for (const mapping of mappings) {
if (!mapping.exportOnly) {
await writeGeneratedPages(mapping);
await importData(mapping.path);
}
}
}
/** Exports every mapping, in config order, replacing each target directory. */
export async function exportMappings(mappings: NoteMapping[], baseUrl: string) {
for (const mapping of mappings) {
await exportData(mapping, baseUrl);
}
}
/**
* Regenerates the Markdown of the mapping's `openApiPages` and `typeDocPages` from their sources,
* and the skill references of the `typeDocPages` that name one.
*/
async function writeGeneratedPages(mapping: NoteMapping) {
const pages = [
...(mapping.openApiPages ?? []).map((page) => ({
noteId: page.noteId,
render: async () => openApiToMarkdown(load(await fs.readFile(page.spec, "utf-8")) as OpenApiSpec)
})),
...(mapping.typeDocPages ?? []).map((page) => ({
noteId: page.noteId,
render: async () => {
const project = await runTypeDoc(page.entryPoint, page.tsconfig);
if (page.skillReference) {
await writeArtifact(page.skillReference, renderSkillReference(project, page));
}
return typeDocToMarkdown(project, page);
}
}))
];
if (!pages.length) {
return;
}
const meta = JSON.parse(await fs.readFile(path.join(mapping.path, "!!!meta.json"), "utf-8")) as NoteMetaFile;
for (const page of pages) {
const found = findDataFile(meta.files, page.noteId);
if (!found?.dataFile.endsWith(".md")) {
throw new Error(`Generated page ${page.noteId} needs a Markdown data file in ${mapping.path}.`);
}
const markdown = await toExportedMarkdown(await page.render(), found.note.title ?? "");
await fs.writeFile(path.join(mapping.path, found.dataFile), markdown);
}
}
/** The skill reference of a TypeDoc page: its declarations under a heading `readSkill()` appends. */
function renderSkillReference(project: Awaited<ReturnType<typeof runTypeDoc>>, page: TypeDocPage) {
return [
`<!-- Generated from ${path.basename(page.entryPoint)} by \`pnpm edit-docs:sync-docs\`. Do not edit. -->`,
"## API reference",
"Every member of `api` and of the objects it returns, as TypeScript declarations, each under the first sentence of its description.",
`\`\`\`ts\n${typeDocToDeclarations(project, page)}\n\`\`\``
].join("\n\n") + "\n";
}
/**
* Converts generated Markdown to the form `MarkdownExportProvider` writes, so an import that no
* export follows leaves the page unchanged on disk.
*/
async function toExportedMarkdown(markdown: string, title: string) {
const { markdownImportService, markdownExportService } = await import("@triliumnext/core");
const html = markdownImportService.renderToHtml(markdown, title);
const exported = markdownExportService.toMarkdown(html);
return exported.startsWith("# ") ? exported : `# ${title}\n${exported}`;
}
/** A note's meta and the path of its data file relative to the export root. */
function findDataFile(
notes: NoteMeta[], noteId: string, dir = ""
): { note: NoteMeta; dataFile: string } | undefined {
for (const note of notes) {
if (note.noteId === noteId) {
return note.dataFileName
? { note, dataFile: path.join(dir, note.dataFileName) }
: undefined;
}
const found = findDataFile(note.children ?? [], noteId, path.join(dir, note.dirFileName ?? ""));
if (found) {
return found;
}
}
return undefined;
}
export async function exportData(mapping: NoteMapping, baseUrl: string) {
const { path: outputPath, ignoredFiles: ignoredFileNames } = mapping;
const ignoredFiles = ignoredFileNames ? new Set(ignoredFileNames) : undefined;
const zipFilePath = "output.zip";
try {
await fs.rm(outputPath, { recursive: true, force: true });
await fs.mkdir(outputPath, { recursive: true });
// First export as zip.
const { zipExportService } = (await import("@triliumnext/core"));
await zipExportService.exportToZipFile(mapping.rootNoteId, mapping.format, zipFilePath, {});
await extractZip(zipFilePath, outputPath, ignoredFiles);
} finally {
await fs.rm(zipFilePath, { force: true });
}
await cleanUpMeta(outputPath, mapping, baseUrl);
}
/**
* Normalizes the freshly exported `!!!meta.json` so it doesn't churn between runs, and — for the
* mapping backing the User Guide — derives the in-app help tree and its rendered content.
*/
async function cleanUpMeta(outputPath: string, mapping: NoteMapping, baseUrl: string) {
const metaPath = path.join(outputPath, "!!!meta.json");
const meta = JSON.parse(await fs.readFile(metaPath, "utf-8")) as NoteMetaFile;
for (const file of meta.files) {
file.notePosition = 1;
traverse(file);
}
function traverse(el: NoteMeta) {
for (const child of el.children || []) {
traverse(child);
}
el.isExpanded = false;
}
await fs.writeFile(metaPath, JSON.stringify(stripAppVersion(meta), null, 4));
if (!mapping.helpMeta && !mapping.helpContent) {
return;
}
const { meta: helpMeta, sources } = buildHelpMeta(meta, baseUrl);
if (mapping.helpMeta) {
await writeArtifact(mapping.helpMeta, serializeHelpMeta(helpMeta));
}
if (mapping.helpContent) {
const { markdownImportService } = await import("@triliumnext/core");
const bundle = buildHelpBundle(
helpMeta,
sources,
(source) => {
try {
return readFileSync(path.join(outputPath, source), "utf-8");
} catch {
return null;
}
},
(markdown, title) => markdownImportService.renderToHtml(markdown, title)
);
// One page per line: the bundle is a flat map from note ID to HTML.
await writeArtifact(mapping.helpContent, JSON.stringify(bundle, null, 4));
}
}
/** Writes a generated help artifact, creating its directory if needed. */
async function writeArtifact(filePath: string, content: string) {
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, content);
}
async function createImportZip(path: string) {
const inputFile = "input.zip";
const archive = new ZipArchive({
zlib: { level: 0 }
});
archive.directory(path, "/");
const outputStream = createWriteStream(inputFile);
archive.pipe(outputStream);
await waitForEnd(archive, outputStream);
try {
return await fs.readFile(inputFile);
} finally {
await fs.rm(inputFile);
}
}
function waitForEnd(archive: Archiver, stream: WriteStream) {
return new Promise<void>((res, rej) => {
stream.on("finish", res);
stream.on("error", rej);
archive.on("error", rej);
archive.finalize().catch(rej);
});
}
export async function createZipFromDirectory(dirPath: string, zipPath: string) {
const archive = new ZipArchive({ zlib: { level: 5 } });
const outputStream = createWriteStream(zipPath);
archive.directory(dirPath, false);
archive.pipe(outputStream);
await waitForEnd(archive, outputStream);
}
export async function extractZip(zipFilePath: string, outputPath: string, ignoredFiles?: Set<string>) {
const promise = deferred<void>();
setTimeout(async () => {
const { getZipProvider } = (await import("@triliumnext/core"));
const zipProvider = getZipProvider();
const buffer = await fs.readFile(zipFilePath);
await zipProvider.readZipFile(buffer, async (entry, readContent) => {
// We ignore directories since they can appear out of order anyway.
if (!entry.fileName.endsWith("/") && !ignoredFiles?.has(entry.fileName)) {
const destPath = path.join(outputPath, entry.fileName);
const fileContent = await readContent();
await fs.mkdir(path.dirname(destPath), { recursive: true });
await fs.writeFile(destPath, normalizeLineEndings(fileContent));
}
});
promise.resolve();
}, 1000);
await promise;
}
/**
* Rewrites CRLF line endings to LF, so that `extractZip` writes the same bytes on Windows as it
* does elsewhere. The exported trees (`docs/`, the demo) are stored with LF, and
* Git only hides the difference once a file is staged.
*
* Binary entries such as images keep their bytes. They are detected the way Git detects them: a
* NUL byte within the first {@link BINARY_DETECTION_LENGTH} bytes marks the content as binary.
*/
export function normalizeLineEndings(content: Uint8Array): Uint8Array {
if (isBinary(content)) {
return content;
}
const normalized = new Uint8Array(content.length);
let length = 0;
for (const [index, byte] of content.entries()) {
if (byte === CARRIAGE_RETURN && content[index + 1] === LINE_FEED) {
continue;
}
normalized[length++] = byte;
}
return length === content.length ? content : normalized.subarray(0, length);
}
const NUL = 0x00;
const LINE_FEED = 0x0a;
const CARRIAGE_RETURN = 0x0d;
const BINARY_DETECTION_LENGTH = 8000;
function isBinary(content: Uint8Array) {
const end = Math.min(content.length, BINARY_DETECTION_LENGTH);
for (let index = 0; index < end; index++) {
if (content[index] === NUL) {
return true;
}
}
return false;
}