Skip to content

Build your personal knowledge base with Trilium Notes

feat(resources): name each resource's directory after its files' hashes

Elian Doran47d34bd1610c
diff --git a/.claude/skills/working-with-transcription/SKILL.md b/.claude/skills/working-with-transcription/SKILL.md
index 75e58b37f7..f4b6ca3183 100644
--- a/.claude/skills/working-with-transcription/SKILL.md
+++ b/.claude/skills/working-with-transcription/SKILL.md
@@ -54,7 +54,7 @@ Nothing the transcription process imports may reach `@triliumnext/core` or `@tri
## The model
-Users download the models from the app: the launcher calls `requireResources(Object.values(TRANSCRIPTION_RESOURCES))` before it creates the meeting note, which shows the download dialog for those missing. They are three resources of the resource manager (`apps/server/src/services/resources.ts`), each in `<data dir>/resources/<id>/<version>/` and checked against the SHA-256 in `model.ts`: `transcription.parakeet` (int8, 670 MB), `transcription.silero-vad` (0.6 MB) and `transcription.titanet` (40 MB). Changing a file of one means a new `version` for it alone, so only that one downloads again. A recording needs Parakeet and Silero; without TitaNet it labels no speakers.
+Users download the models from the app: the launcher calls `requireResources(Object.values(TRANSCRIPTION_RESOURCES))` before it creates the meeting note, which shows the download dialog for those missing. They are three resources of the resource manager (`apps/server/src/services/resources.ts`), each in `<data dir>/resources/<id>-<hash>/` and checked against the SHA-256 in `model.ts`: `transcription.parakeet` (int8, 670 MB), `transcription.silero-vad` (0.6 MB) and `transcription.titanet` (40 MB). The hash in the directory name covers every file's path and SHA-256, so changing a file of one model downloads that model alone again, into a new directory. A recording needs Parakeet and Silero; without TitaNet it labels no speakers.
A directory installed by hand, `<data dir>/transcription-model` (`LOCAL_MODEL_DIR`), takes precedence over the downloads, each model for itself (Silero when `silero_vad.onnx` is there, TitaNet when `speaker/nemo_en_titanet_small.onnx` is); for `pnpm desktop:start` that is `apps/desktop/data/transcription-model`, which the scripts below read. Parakeet comes in two precisions, and `loadSherpaEngine()` takes the full-precision one when `encoder.onnx` is present, else the int8 one; it records which as `precision` in the engine settings and applies `blankPenalty` to int8 only (see "Findings so far").
diff --git a/apps/desktop/src/services/transcription/model.ts b/apps/desktop/src/services/transcription/model.ts
index 7dee9d2ece..37baa849e3 100644
--- a/apps/desktop/src/services/transcription/model.ts
+++ b/apps/desktop/src/services/transcription/model.ts
@@ -22,7 +22,6 @@ const SHERPA_RELEASES = "https://github.com/k2-fsa/sherpa-onnx/releases/download
/** Parakeet TDT 0.6B v3, int8: the files `loadSherpaEngine()` reads from its recognizer directory. */
export const RECOGNIZER_MODEL: ResourceDefinition = {
id: TRANSCRIPTION_RESOURCES.recognizer,
- version: "v3-int8-2bda32ec",
title: () => t("transcription.recognizer-title"),
description: () => t("transcription.recognizer-description"),
icon: "bx bx-captions",
@@ -58,7 +57,6 @@ export const RECOGNIZER_MODEL: ResourceDefinition = {
export const VAD_MODEL: ResourceDefinition = {
id: TRANSCRIPTION_RESOURCES.vad,
- version: "1",
title: () => t("transcription.vad-title"),
description: () => t("transcription.vad-description"),
icon: "bx bx-pulse",
@@ -74,7 +72,6 @@ export const VAD_MODEL: ResourceDefinition = {
export const SPEAKER_MODEL: ResourceDefinition = {
id: TRANSCRIPTION_RESOURCES.speaker,
- version: "1",
title: () => t("transcription.speaker-title"),
description: () => t("transcription.speaker-description"),
icon: "bx bx-user-voice",
diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.ts
index 19ba98e5dd..675a6d44e7 100644
--- a/apps/server/src/services/resources.spec.ts
+++ b/apps/server/src/services/resources.spec.ts
@@ -78,7 +78,8 @@ describe("ResourceManager", () => {
file: null
});
expect(reports.some(({ file }) => file === "model.bin")).toBe(true);
- const directory = path.join(baseDir, "test.model", "1");
+ const directory = resourceDirectory();
+ expect(path.basename(directory)).toMatch(/^test\.model-[0-9a-f]{12}$/);
expect(manager.getPath("test.model")).toBe(directory);
expect(fs.readFileSync(path.join(directory, "nested", "model.bin")).equals(MODEL)).toBe(true);
expect(fs.readdirSync(path.join(directory, "nested"))).toEqual([ "model.bin" ]);
@@ -108,8 +109,7 @@ describe("ResourceManager", () => {
expect(failed.status).toBe("failed");
expect(failed.error).toBeTruthy();
expect(manager.getPath("test.model")).toBeNull();
- const directory = path.join(baseDir, "test.model", "1");
- expect(fs.readdirSync(directory)).toEqual([ "nested" ]);
+ expect(fs.readdirSync(resourceDirectory())).toEqual([ ".verified.json", "nested" ]);
const [ info ] = (await manager.getStatus([ "test.model" ])).resources;
expect(info).toMatchObject({ installed: false, downloadedSize: MODEL.length });
@@ -143,11 +143,54 @@ describe("ResourceManager", () => {
manager.cancelDownload(first.downloadId);
expect((await waitForEnd()).status).toBe("canceled");
- const directory = path.join(baseDir, "test.model", "1");
- expect(fs.readdirSync(directory)).toEqual([ "nested" ]);
+ expect(fs.readdirSync(resourceDirectory())).toEqual([ ".verified.json", "nested" ]);
expect((await manager.getStatus([ "test.model" ])).download).toBeNull();
});
+ it("downloads into a new directory once a file of the resource changes", async () => {
+ manager.register(defineModel());
+ manager.startDownload([ "test.model" ]);
+ await waitForEnd();
+
+ const updated = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));
+ updated.register(defineModel({ tokensHash: sha256(Buffer.from("a\nb\nd\n")) }));
+ expect(updated.getPath("test.model")).toBeNull();
+ expect((await updated.getStatus([ "test.model" ])).resources[0].downloadedSize).toBe(0);
+ });
+
+ it("notices a file changed after it was installed, and downloads only that one again", async () => {
+ manager.register(defineModel());
+ manager.startDownload([ "test.model" ]);
+ await waitForEnd();
+ const tokens = path.join(resourceDirectory(), "tokens.txt");
+
+ fs.writeFileSync(tokens, "x\ny\nz\n");
+ fs.utimesSync(tokens, new Date(2020, 0, 1), new Date(2020, 0, 1));
+ expect(manager.getPath("test.model")).toBeNull();
+ expect((await manager.getStatus([ "test.model" ])).resources[0].downloadedSize)
+ .toBe(MODEL.length);
+
+ requested.length = 0;
+ manager.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).status).toBe("succeeded");
+ expect(requested).toEqual([ "/tokens.txt" ]);
+ expect(fs.readFileSync(tokens).equals(TOKENS)).toBe(true);
+ });
+
+ it("adopts files put in place by hand once their hash matches, without downloading them", async () => {
+ manager.register(defineModel());
+ manager.startDownload([ "test.model" ]);
+ await waitForEnd();
+ fs.rmSync(path.join(resourceDirectory(), ".verified.json"));
+ expect(manager.getPath("test.model")).toBeNull();
+
+ requested.length = 0;
+ manager.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).status).toBe("succeeded");
+ expect(requested).toEqual([]);
+ expect(manager.getPath("test.model")).toBe(resourceDirectory());
+ });
+
it("prefers a local copy, and refuses invalid or unknown resources", () => {
manager.register({ ...defineModel(), findLocalCopy: () => "/hand/installed" });
expect(manager.getPath("test.model")).toBe("/hand/installed");
@@ -165,7 +208,6 @@ function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash
tokensSize = TOKENS.length } = {}): ResourceDefinition {
return {
id,
- version: "1",
title: () => "Test model",
description: () => "For tests",
icon: "bx bx-test-tube",
@@ -177,6 +219,13 @@ function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash
};
}
+/** The directory the resource `id` was downloaded into: the only one named after it. */
+function resourceDirectory(id = "test.model") {
+ const directories = fs.readdirSync(baseDir).filter((name) => name.startsWith(`${id}-`));
+ expect(directories).toHaveLength(1);
+ return path.join(baseDir, directories[0]);
+}
+
function file(filePath: string, content: Buffer): ResourceFile {
return {
path: filePath,
diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.ts
index c0a3e5907d..6610d4b7b6 100644
--- a/apps/server/src/services/resources.ts
+++ b/apps/server/src/services/resources.ts
@@ -23,8 +23,6 @@ export interface ResourceFile {
export interface ResourceDefinition {
/** Lowercase words separated by dots, the feature first: `transcription.model`. */
id: string;
- /** Part of the resource's directory, so a new version downloads next to the old one. */
- version: string;
title(): string;
description(): string;
/** Boxicons classes, `bx bx-…`, for the download dialog. */
@@ -44,20 +42,27 @@ interface Download {
controller: AbortController;
}
+/** The size and modification time of each verified file of a resource, by its path. */
+type VerifiedFiles = Record<string, { size: number; mtimeMs: number }>;
+
const ID_PATTERN = /^[a-z0-9-]+(\.[a-z0-9-]+)*$/;
-const VERSION_PATTERN = /^[\w.-]+$/;
const PROGRESS_INTERVAL_MS = 250;
-/** Written into a resource's directory once every file in it is verified. */
-const COMPLETE_MARKER = ".complete";
+/** How many hex digits of the files' combined hash a resource's directory name carries. */
+const CONTENT_HASH_LENGTH = 12;
+/** Holds the {@link VerifiedFiles} of a resource, in its directory. */
+const VERIFIED_FILE = ".verified.json";
/**
- * Downloads the resources features register into `baseDir`, one directory per resource and
- * version. A file is written under a `.partial` name and renamed once its size and SHA-256 match,
- * so a file under its own name is always verified, and an interrupted download keeps the files it
- * finished.
+ * Downloads the resources features register into `baseDir`, one directory per resource named
+ * `<id>-<hash>`, the hash covering every file's path and SHA-256: a resource whose files change
+ * downloads into a new directory. A file is written under a `.partial` name and renamed once its
+ * size and SHA-256 match, then recorded in `.verified.json` with its size and modification time.
+ * A file counts as installed only while it still matches that record, so one changed after its
+ * download is downloaded again, and an interrupted download keeps the files it finished.
*/
export class ResourceManager {
private readonly definitions = new Map<string, ResourceDefinition>();
+ private readonly directories = new Map<string, string>();
private readonly downloads = new Map<string, Download>();
constructor(
@@ -72,6 +77,8 @@ export class ResourceManager {
throw new Error(`The resource ${definition.id} is already registered.`);
}
this.definitions.set(definition.id, definition);
+ this.directories.set(definition.id, path.join(this.baseDir,
+ `${definition.id}-${getContentHash(definition.files)}`));
}
}
@@ -83,7 +90,10 @@ export class ResourceManager {
return localCopy;
}
const directory = this.getDirectory(definition);
- return fs.existsSync(path.join(directory, COMPLETE_MARKER)) ? directory : null;
+ const verified = readVerifiedFiles(directory);
+ const isInstalled = definition.files
+ .every((file) => isVerified(verified, path.join(directory, file.path), file));
+ return isInstalled ? directory : null;
}
async getStatus(ids: string[]): Promise<ResourcesStatusResponse> {
@@ -143,21 +153,28 @@ export class ResourceManager {
try {
for (const definition of definitions) {
const directory = this.getDirectory(definition);
+ const verified = readVerifiedFiles(directory);
for (const file of definition.files) {
const destination = path.join(directory, file.path);
- if (isDownloaded(destination, file)) {
+ if (isVerified(verified, destination, file)) {
progress.receivedBytes += file.size;
continue;
}
progress.file = path.basename(file.path);
report(true);
await fs.promises.mkdir(path.dirname(destination), { recursive: true });
- await downloadFile(file, destination, controller.signal, (bytes) => {
- progress.receivedBytes += bytes;
- report(false);
- });
+ if (await hasContent(destination, file)) {
+ progress.receivedBytes += file.size;
+ } else {
+ await downloadFile(file, destination, controller.signal, (bytes) => {
+ progress.receivedBytes += bytes;
+ report(false);
+ });
+ }
+ const { size, mtimeMs } = await fs.promises.stat(destination);
+ verified[file.path] = { size, mtimeMs };
+ await writeVerifiedFiles(directory, verified);
}
- await fs.promises.writeFile(path.join(directory, COMPLETE_MARKER), "");
getLog().info(`Resources: installed ${definition.id} in ${directory}.`);
}
progress.status = "succeeded";
@@ -185,13 +202,18 @@ export class ResourceManager {
}
private getDirectory(definition: ResourceDefinition) {
- return path.join(this.baseDir, definition.id, definition.version);
+ const directory = this.directories.get(definition.id);
+ if (!directory) {
+ throw new Error(`The resource ${definition.id} is not registered.`);
+ }
+ return directory;
}
private getInfo(definition: ResourceDefinition): ResourceInfo {
const directory = this.getDirectory(definition);
+ const verified = readVerifiedFiles(directory);
const downloaded = definition.files
- .filter((file) => isDownloaded(path.join(directory, file.path), file));
+ .filter((file) => isVerified(verified, path.join(directory, file.path), file));
return {
id: definition.id,
title: definition.title(),
@@ -269,9 +291,9 @@ export async function downloadFile(file: ResourceFile, destination: string, sign
}
}
-function validateDefinition({ id, version, files }: ResourceDefinition) {
- if (!ID_PATTERN.test(id) || !VERSION_PATTERN.test(version)) {
- throw new Error(`The resource ${id} has an invalid ID or version.`);
+function validateDefinition({ id, files }: ResourceDefinition) {
+ if (!ID_PATTERN.test(id)) {
+ throw new Error(`The resource ${id} has an invalid ID.`);
}
for (const file of files) {
const segments = file.path.split("/");
@@ -281,13 +303,57 @@ function validateDefinition({ id, version, files }: ResourceDefinition) {
}
}
-/** Whether `destination` is a finished download of `file`: only a verified file gets its name. */
-function isDownloaded(destination: string, file: ResourceFile) {
+/** The start of a SHA-256 over every file's path and hash, which names a resource's directory. */
+function getContentHash(files: ResourceFile[]) {
+ const hash = createHash("sha256");
+ const sorted = [ ...files ].sort((a, b) => a.path.localeCompare(b.path));
+ for (const file of sorted) {
+ hash.update(`${file.path}\0${file.sha256}\n`);
+ }
+ return hash.digest("hex").slice(0, CONTENT_HASH_LENGTH);
+}
+
+function readVerifiedFiles(directory: string): VerifiedFiles {
try {
- return fs.statSync(destination).size === file.size;
+ return JSON.parse(fs.readFileSync(path.join(directory, VERIFIED_FILE), "utf8"));
+ } catch {
+ return {};
+ }
+}
+
+/** Writes `verified` through a temporary file, so an interruption never leaves a torn record. */
+async function writeVerifiedFiles(directory: string, verified: VerifiedFiles) {
+ const destination = path.join(directory, VERIFIED_FILE);
+ await fs.promises.writeFile(`${destination}.partial`, JSON.stringify(verified));
+ await fs.promises.rename(`${destination}.partial`, destination);
+}
+
+/** Whether `destination` is `file` as it was verified: same size and modification time. */
+function isVerified(verified: VerifiedFiles, destination: string, file: ResourceFile) {
+ const record = verified[file.path];
+ if (!record || record.size !== file.size) {
+ return false;
+ }
+ try {
+ const { size, mtimeMs } = fs.statSync(destination);
+ return size === record.size && mtimeMs === record.mtimeMs;
+ } catch {
+ return false;
+ }
+}
+
+/** Whether `destination` already holds `file`, as one put there by hand: checked by its hash. */
+async function hasContent(destination: string, file: ResourceFile) {
+ try {
+ if ((await fs.promises.stat(destination)).size !== file.size) {
+ return false;
+ }
} catch {
return false;
}
+ const hash = createHash("sha256");
+ await pipeline(fs.createReadStream(destination), hash);
+ return hash.digest("hex") === file.sha256;
}
function sumSizes(files: ResourceFile[]) {