Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(resources): name each resource's directory after its files' hashes
Elian Doran47d34bd1610c
diff --git a/.claude/skills/working-with-transcription/SKILL.md b/.claude/skills/working-with-transcription/SKILL.mdindex 75e58b37f7..f4b6ca3183 100644--- a/.claude/skills/working-with-transcription/SKILL.md+++ b/.claude/skills/working-with-transcription/SKILL.md@@ -54,7 +54,7 @@ Nothing the transcription process imports may reach `@triliumnext/core` or `@tri## The model-Users download the models from the app: the launcher calls `requireResources(Object.values(TRANSCRIPTION_RESOURCES))` before it creates the meeting note, which shows the download dialog for those missing. They are three resources of the resource manager (`apps/server/src/services/resources.ts`), each in `<data dir>/resources/<id>/<version>/` and checked against the SHA-256 in `model.ts`: `transcription.parakeet` (int8, 670 MB), `transcription.silero-vad` (0.6 MB) and `transcription.titanet` (40 MB). Changing a file of one means a new `version` for it alone, so only that one downloads again. A recording needs Parakeet and Silero; without TitaNet it labels no speakers.+Users download the models from the app: the launcher calls `requireResources(Object.values(TRANSCRIPTION_RESOURCES))` before it creates the meeting note, which shows the download dialog for those missing. They are three resources of the resource manager (`apps/server/src/services/resources.ts`), each in `<data dir>/resources/<id>-<hash>/` and checked against the SHA-256 in `model.ts`: `transcription.parakeet` (int8, 670 MB), `transcription.silero-vad` (0.6 MB) and `transcription.titanet` (40 MB). The hash in the directory name covers every file's path and SHA-256, so changing a file of one model downloads that model alone again, into a new directory. A recording needs Parakeet and Silero; without TitaNet it labels no speakers.A directory installed by hand, `<data dir>/transcription-model` (`LOCAL_MODEL_DIR`), takes precedence over the downloads, each model for itself (Silero when `silero_vad.onnx` is there, TitaNet when `speaker/nemo_en_titanet_small.onnx` is); for `pnpm desktop:start` that is `apps/desktop/data/transcription-model`, which the scripts below read. Parakeet comes in two precisions, and `loadSherpaEngine()` takes the full-precision one when `encoder.onnx` is present, else the int8 one; it records which as `precision` in the engine settings and applies `blankPenalty` to int8 only (see "Findings so far").diff --git a/apps/desktop/src/services/transcription/model.ts b/apps/desktop/src/services/transcription/model.tsindex 7dee9d2ece..37baa849e3 100644--- a/apps/desktop/src/services/transcription/model.ts+++ b/apps/desktop/src/services/transcription/model.ts@@ -22,7 +22,6 @@ const SHERPA_RELEASES = "https://github.com/k2-fsa/sherpa-onnx/releases/download/** Parakeet TDT 0.6B v3, int8: the files `loadSherpaEngine()` reads from its recognizer directory. */export const RECOGNIZER_MODEL: ResourceDefinition = {id: TRANSCRIPTION_RESOURCES.recognizer,- version: "v3-int8-2bda32ec",title: () => t("transcription.recognizer-title"),description: () => t("transcription.recognizer-description"),icon: "bx bx-captions",@@ -58,7 +57,6 @@ export const RECOGNIZER_MODEL: ResourceDefinition = {export const VAD_MODEL: ResourceDefinition = {id: TRANSCRIPTION_RESOURCES.vad,- version: "1",title: () => t("transcription.vad-title"),description: () => t("transcription.vad-description"),icon: "bx bx-pulse",@@ -74,7 +72,6 @@ export const VAD_MODEL: ResourceDefinition = {export const SPEAKER_MODEL: ResourceDefinition = {id: TRANSCRIPTION_RESOURCES.speaker,- version: "1",title: () => t("transcription.speaker-title"),description: () => t("transcription.speaker-description"),icon: "bx bx-user-voice",diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.tsindex 19ba98e5dd..675a6d44e7 100644--- a/apps/server/src/services/resources.spec.ts+++ b/apps/server/src/services/resources.spec.ts@@ -78,7 +78,8 @@ describe("ResourceManager", () => {file: null});expect(reports.some(({ file }) => file === "model.bin")).toBe(true);- const directory = path.join(baseDir, "test.model", "1");+ const directory = resourceDirectory();+ expect(path.basename(directory)).toMatch(/^test\.model-[0-9a-f]{12}$/);expect(manager.getPath("test.model")).toBe(directory);expect(fs.readFileSync(path.join(directory, "nested", "model.bin")).equals(MODEL)).toBe(true);expect(fs.readdirSync(path.join(directory, "nested"))).toEqual([ "model.bin" ]);@@ -108,8 +109,7 @@ describe("ResourceManager", () => {expect(failed.status).toBe("failed");expect(failed.error).toBeTruthy();expect(manager.getPath("test.model")).toBeNull();- const directory = path.join(baseDir, "test.model", "1");- expect(fs.readdirSync(directory)).toEqual([ "nested" ]);+ expect(fs.readdirSync(resourceDirectory())).toEqual([ ".verified.json", "nested" ]);const [ info ] = (await manager.getStatus([ "test.model" ])).resources;expect(info).toMatchObject({ installed: false, downloadedSize: MODEL.length });@@ -143,11 +143,54 @@ describe("ResourceManager", () => {manager.cancelDownload(first.downloadId);expect((await waitForEnd()).status).toBe("canceled");- const directory = path.join(baseDir, "test.model", "1");- expect(fs.readdirSync(directory)).toEqual([ "nested" ]);+ expect(fs.readdirSync(resourceDirectory())).toEqual([ ".verified.json", "nested" ]);expect((await manager.getStatus([ "test.model" ])).download).toBeNull();});+ it("downloads into a new directory once a file of the resource changes", async () => {+ manager.register(defineModel());+ manager.startDownload([ "test.model" ]);+ await waitForEnd();++ const updated = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));+ updated.register(defineModel({ tokensHash: sha256(Buffer.from("a\nb\nd\n")) }));+ expect(updated.getPath("test.model")).toBeNull();+ expect((await updated.getStatus([ "test.model" ])).resources[0].downloadedSize).toBe(0);+ });++ it("notices a file changed after it was installed, and downloads only that one again", async () => {+ manager.register(defineModel());+ manager.startDownload([ "test.model" ]);+ await waitForEnd();+ const tokens = path.join(resourceDirectory(), "tokens.txt");++ fs.writeFileSync(tokens, "x\ny\nz\n");+ fs.utimesSync(tokens, new Date(2020, 0, 1), new Date(2020, 0, 1));+ expect(manager.getPath("test.model")).toBeNull();+ expect((await manager.getStatus([ "test.model" ])).resources[0].downloadedSize)+ .toBe(MODEL.length);++ requested.length = 0;+ manager.startDownload([ "test.model" ]);+ expect((await waitForEnd()).status).toBe("succeeded");+ expect(requested).toEqual([ "/tokens.txt" ]);+ expect(fs.readFileSync(tokens).equals(TOKENS)).toBe(true);+ });++ it("adopts files put in place by hand once their hash matches, without downloading them", async () => {+ manager.register(defineModel());+ manager.startDownload([ "test.model" ]);+ await waitForEnd();+ fs.rmSync(path.join(resourceDirectory(), ".verified.json"));+ expect(manager.getPath("test.model")).toBeNull();++ requested.length = 0;+ manager.startDownload([ "test.model" ]);+ expect((await waitForEnd()).status).toBe("succeeded");+ expect(requested).toEqual([]);+ expect(manager.getPath("test.model")).toBe(resourceDirectory());+ });+it("prefers a local copy, and refuses invalid or unknown resources", () => {manager.register({ ...defineModel(), findLocalCopy: () => "/hand/installed" });expect(manager.getPath("test.model")).toBe("/hand/installed");@@ -165,7 +208,6 @@ function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHashtokensSize = TOKENS.length } = {}): ResourceDefinition {return {id,- version: "1",title: () => "Test model",description: () => "For tests",icon: "bx bx-test-tube",@@ -177,6 +219,13 @@ function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash};}+/** The directory the resource `id` was downloaded into: the only one named after it. */+function resourceDirectory(id = "test.model") {+ const directories = fs.readdirSync(baseDir).filter((name) => name.startsWith(`${id}-`));+ expect(directories).toHaveLength(1);+ return path.join(baseDir, directories[0]);+}+function file(filePath: string, content: Buffer): ResourceFile {return {path: filePath,diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.tsindex c0a3e5907d..6610d4b7b6 100644--- a/apps/server/src/services/resources.ts+++ b/apps/server/src/services/resources.ts@@ -23,8 +23,6 @@ export interface ResourceFile {export interface ResourceDefinition {/** Lowercase words separated by dots, the feature first: `transcription.model`. */id: string;- /** Part of the resource's directory, so a new version downloads next to the old one. */- version: string;title(): string;description(): string;/** Boxicons classes, `bx bx-…`, for the download dialog. */@@ -44,20 +42,27 @@ interface Download {controller: AbortController;}+/** The size and modification time of each verified file of a resource, by its path. */+type VerifiedFiles = Record<string, { size: number; mtimeMs: number }>;+const ID_PATTERN = /^[a-z0-9-]+(\.[a-z0-9-]+)*$/;-const VERSION_PATTERN = /^[\w.-]+$/;const PROGRESS_INTERVAL_MS = 250;-/** Written into a resource's directory once every file in it is verified. */-const COMPLETE_MARKER = ".complete";+/** How many hex digits of the files' combined hash a resource's directory name carries. */+const CONTENT_HASH_LENGTH = 12;+/** Holds the {@link VerifiedFiles} of a resource, in its directory. */+const VERIFIED_FILE = ".verified.json";/**- * Downloads the resources features register into `baseDir`, one directory per resource and- * version. A file is written under a `.partial` name and renamed once its size and SHA-256 match,- * so a file under its own name is always verified, and an interrupted download keeps the files it- * finished.+ * Downloads the resources features register into `baseDir`, one directory per resource named+ * `<id>-<hash>`, the hash covering every file's path and SHA-256: a resource whose files change+ * downloads into a new directory. A file is written under a `.partial` name and renamed once its+ * size and SHA-256 match, then recorded in `.verified.json` with its size and modification time.+ * A file counts as installed only while it still matches that record, so one changed after its+ * download is downloaded again, and an interrupted download keeps the files it finished.*/export class ResourceManager {private readonly definitions = new Map<string, ResourceDefinition>();+ private readonly directories = new Map<string, string>();private readonly downloads = new Map<string, Download>();constructor(@@ -72,6 +77,8 @@ export class ResourceManager {throw new Error(`The resource ${definition.id} is already registered.`);}this.definitions.set(definition.id, definition);+ this.directories.set(definition.id, path.join(this.baseDir,+ `${definition.id}-${getContentHash(definition.files)}`));}}@@ -83,7 +90,10 @@ export class ResourceManager {return localCopy;}const directory = this.getDirectory(definition);- return fs.existsSync(path.join(directory, COMPLETE_MARKER)) ? directory : null;+ const verified = readVerifiedFiles(directory);+ const isInstalled = definition.files+ .every((file) => isVerified(verified, path.join(directory, file.path), file));+ return isInstalled ? directory : null;}async getStatus(ids: string[]): Promise<ResourcesStatusResponse> {@@ -143,21 +153,28 @@ export class ResourceManager {try {for (const definition of definitions) {const directory = this.getDirectory(definition);+ const verified = readVerifiedFiles(directory);for (const file of definition.files) {const destination = path.join(directory, file.path);- if (isDownloaded(destination, file)) {+ if (isVerified(verified, destination, file)) {progress.receivedBytes += file.size;continue;}progress.file = path.basename(file.path);report(true);await fs.promises.mkdir(path.dirname(destination), { recursive: true });- await downloadFile(file, destination, controller.signal, (bytes) => {- progress.receivedBytes += bytes;- report(false);- });+ if (await hasContent(destination, file)) {+ progress.receivedBytes += file.size;+ } else {+ await downloadFile(file, destination, controller.signal, (bytes) => {+ progress.receivedBytes += bytes;+ report(false);+ });+ }+ const { size, mtimeMs } = await fs.promises.stat(destination);+ verified[file.path] = { size, mtimeMs };+ await writeVerifiedFiles(directory, verified);}- await fs.promises.writeFile(path.join(directory, COMPLETE_MARKER), "");getLog().info(`Resources: installed ${definition.id} in ${directory}.`);}progress.status = "succeeded";@@ -185,13 +202,18 @@ export class ResourceManager {}private getDirectory(definition: ResourceDefinition) {- return path.join(this.baseDir, definition.id, definition.version);+ const directory = this.directories.get(definition.id);+ if (!directory) {+ throw new Error(`The resource ${definition.id} is not registered.`);+ }+ return directory;}private getInfo(definition: ResourceDefinition): ResourceInfo {const directory = this.getDirectory(definition);+ const verified = readVerifiedFiles(directory);const downloaded = definition.files- .filter((file) => isDownloaded(path.join(directory, file.path), file));+ .filter((file) => isVerified(verified, path.join(directory, file.path), file));return {id: definition.id,title: definition.title(),@@ -269,9 +291,9 @@ export async function downloadFile(file: ResourceFile, destination: string, sign}}-function validateDefinition({ id, version, files }: ResourceDefinition) {- if (!ID_PATTERN.test(id) || !VERSION_PATTERN.test(version)) {- throw new Error(`The resource ${id} has an invalid ID or version.`);+function validateDefinition({ id, files }: ResourceDefinition) {+ if (!ID_PATTERN.test(id)) {+ throw new Error(`The resource ${id} has an invalid ID.`);}for (const file of files) {const segments = file.path.split("/");@@ -281,13 +303,57 @@ function validateDefinition({ id, version, files }: ResourceDefinition) {}}-/** Whether `destination` is a finished download of `file`: only a verified file gets its name. */-function isDownloaded(destination: string, file: ResourceFile) {+/** The start of a SHA-256 over every file's path and hash, which names a resource's directory. */+function getContentHash(files: ResourceFile[]) {+ const hash = createHash("sha256");+ const sorted = [ ...files ].sort((a, b) => a.path.localeCompare(b.path));+ for (const file of sorted) {+ hash.update(`${file.path}\0${file.sha256}\n`);+ }+ return hash.digest("hex").slice(0, CONTENT_HASH_LENGTH);+}++function readVerifiedFiles(directory: string): VerifiedFiles {try {- return fs.statSync(destination).size === file.size;+ return JSON.parse(fs.readFileSync(path.join(directory, VERIFIED_FILE), "utf8"));+ } catch {+ return {};+ }+}++/** Writes `verified` through a temporary file, so an interruption never leaves a torn record. */+async function writeVerifiedFiles(directory: string, verified: VerifiedFiles) {+ const destination = path.join(directory, VERIFIED_FILE);+ await fs.promises.writeFile(`${destination}.partial`, JSON.stringify(verified));+ await fs.promises.rename(`${destination}.partial`, destination);+}++/** Whether `destination` is `file` as it was verified: same size and modification time. */+function isVerified(verified: VerifiedFiles, destination: string, file: ResourceFile) {+ const record = verified[file.path];+ if (!record || record.size !== file.size) {+ return false;+ }+ try {+ const { size, mtimeMs } = fs.statSync(destination);+ return size === record.size && mtimeMs === record.mtimeMs;+ } catch {+ return false;+ }+}++/** Whether `destination` already holds `file`, as one put there by hand: checked by its hash. */+async function hasContent(destination: string, file: ResourceFile) {+ try {+ if ((await fs.promises.stat(destination)).size !== file.size) {+ return false;+ }} catch {return false;}+ const hash = createHash("sha256");+ await pipeline(fs.createReadStream(destination), hash);+ return hash.digest("hex") === file.sha256;}function sumSizes(files: ResourceFile[]) {