Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(resources): resume an interrupted download from the bytes it kept
Elian Dorancff77e3bfd7b
diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.tsindex 675a6d44e7..732fa1a5c2 100644--- a/apps/server/src/services/resources.spec.ts+++ b/apps/server/src/services/resources.spec.ts@@ -15,6 +15,10 @@ const TOKENS = Buffer.from("a\nb\nc\n");let server: http.Server;let baseUrl = "";const requested: string[] = [];+/** The `Range` header of each request, in order. */+const ranges: (string | undefined)[] = [];+/** Makes the next whole-file request for `/model.bin` drop the connection after this many bytes. */+let dropModelAfter: number | null = null;/** Requests to `/stall`, answered with headers and no body until the test ends. */let stalled: http.ServerResponse[] = [];@@ -27,7 +31,13 @@ let manager: ResourceManager;beforeAll(async () => {server = http.createServer((req, res) => {requested.push(req.url ?? "");+ ranges.push(req.headers.range);if (req.url === "/model.bin") {+ serveModel(req, res);+ } else if (req.url === "/model-no-range.bin") {+ res.end(MODEL);+ } else if (req.url === "/model-wrong-range.bin") {+ res.writeHead(206, { "Content-Range": `bytes 0-${MODEL.length - 1}/${MODEL.length}` });res.end(MODEL);} else if (req.url === "/tokens.txt") {res.end(TOKENS);@@ -49,6 +59,8 @@ beforeEach(() => {baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-resources-"));broadcasts = [];requested.length = 0;+ ranges.length = 0;+ dropModelAfter = null;manager = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));});@@ -133,7 +145,7 @@ describe("ResourceManager", () => {expect(manager.getPath("test.short")).toBeNull();});- it("joins a running download, and cancels it without leaving a partial file", async () => {+ it("joins a running download, and cancels it without leaving an empty partial file", async () => {manager.register(defineModel({ tokensUrl: "/stall" }));const first = manager.startDownload([ "test.model" ]);@@ -191,6 +203,79 @@ describe("ResourceManager", () => {expect(manager.getPath("test.model")).toBe(resourceDirectory());});+ it("resumes a file after the connection drops, asking only for the bytes it is missing", async () => {+ manager.register(defineModel());+ dropModelAfter = 120_000;+ manager.startDownload([ "test.model" ]);+ expect((await waitForEnd()).status).toBe("failed");++ const partial = path.join(resourceDirectory(), "nested", "model.bin.partial");+ const kept = fs.statSync(partial).size;+ expect(kept).toBeGreaterThan(0);+ expect(kept).toBeLessThanOrEqual(120_000);+ expect((await manager.getStatus([ "test.model" ])).resources[0].downloadedSize).toBe(kept);++ ranges.length = 0;+ manager.startDownload([ "test.model" ]);+ expect(await waitForEnd()).toMatchObject({+ status: "succeeded", receivedBytes: MODEL.length + TOKENS.length+ });+ expect(ranges[0]).toBe(`bytes=${kept}-`);+ expect(fs.readFileSync(path.join(resourceDirectory(), "nested", "model.bin")).equals(MODEL))+ .toBe(true);+ });++ it("starts over when the server sends the whole file instead of the rest", async () => {+ manager.register(defineModel());+ dropModelAfter = 120_000;+ manager.startDownload([ "test.model" ]);+ await waitForEnd();++ const resumed = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));+ resumed.register(defineModel({ modelUrl: "/model-no-range.bin" }));+ ranges.length = 0;+ resumed.startDownload([ "test.model" ]);+ expect(await waitForEnd()).toMatchObject({+ status: "succeeded", receivedBytes: MODEL.length + TOKENS.length+ });+ expect(ranges[0]?.startsWith("bytes=")).toBe(true);+ expect(fs.readFileSync(path.join(resourceDirectory(), "nested", "model.bin")).equals(MODEL))+ .toBe(true);+ });++ it("discards the kept bytes when the server answers with another range than asked", async () => {+ manager.register(defineModel());+ dropModelAfter = 120_000;+ manager.startDownload([ "test.model" ]);+ await waitForEnd();+ const partial = path.join(resourceDirectory(), "nested", "model.bin.partial");+ expect(fs.existsSync(partial)).toBe(true);++ const resumed = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));+ resumed.register(defineModel({ modelUrl: "/model-wrong-range.bin" }));+ resumed.startDownload([ "test.model" ]);+ expect((await waitForEnd()).status).toBe("failed");+ expect(fs.existsSync(partial)).toBe(false);+ });++ it("discards kept bytes that turn out wrong, and downloads the whole file on the next try", async () => {+ manager.register(defineModel());+ dropModelAfter = 120_000;+ manager.startDownload([ "test.model" ]);+ await waitForEnd();+ const partial = path.join(resourceDirectory(), "nested", "model.bin.partial");+ fs.writeFileSync(partial, Buffer.alloc(fs.statSync(partial).size, "x"));++ manager.startDownload([ "test.model" ]);+ expect((await waitForEnd()).status).toBe("failed");+ expect(fs.existsSync(partial)).toBe(false);++ ranges.length = 0;+ manager.startDownload([ "test.model" ]);+ expect((await waitForEnd()).status).toBe("succeeded");+ expect(ranges[0]).toBeUndefined();+ });+it("prefers a local copy, and refuses invalid or unknown resources", () => {manager.register({ ...defineModel(), findLocalCopy: () => "/hand/installed" });expect(manager.getPath("test.model")).toBe("/hand/installed");@@ -204,8 +289,8 @@ describe("ResourceManager", () => {});});-function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash = sha256(TOKENS),- tokensSize = TOKENS.length } = {}): ResourceDefinition {+function defineModel({ id = "test.model", modelUrl = "/model.bin", tokensUrl = "/tokens.txt",+ tokensHash = sha256(TOKENS), tokensSize = TOKENS.length } = {}): ResourceDefinition {return {id,title: () => "Test model",@@ -213,12 +298,32 @@ function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHashicon: "bx bx-test-tube",license: "MIT",files: [- file("nested/model.bin", MODEL),+ { ...file("nested/model.bin", MODEL), url: `${baseUrl}${modelUrl}` },{ path: "tokens.txt", url: `${baseUrl}${tokensUrl}`, size: tokensSize, sha256: tokensHash }]};}+/**+ * Serves the model, from the offset a `Range: bytes=<n>-` header asks for when there is one, or+ * drops the connection after {@link dropModelAfter} bytes once when that is set.+ */+function serveModel(req: http.IncomingMessage, res: http.ServerResponse) {+ const range = req.headers.range;+ if (range?.startsWith("bytes=")) {+ const start = Number(range.slice("bytes=".length, range.indexOf("-")));+ res.writeHead(206, { "Content-Range": `bytes ${start}-${MODEL.length - 1}/${MODEL.length}` });+ res.end(MODEL.subarray(start));+ } else if (dropModelAfter !== null) {+ const length = dropModelAfter;+ dropModelAfter = null;+ res.writeHead(200, { "Content-Length": MODEL.length });+ res.write(MODEL.subarray(0, length), () => setTimeout(() => res.destroy(), 20));+ } else {+ res.end(MODEL);+ }+}+/** The directory the resource `id` was downloaded into: the only one named after it. */function resourceDirectory(id = "test.model") {const directories = fs.readdirSync(baseDir).filter((name) => name.startsWith(`${id}-`));diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.tsindex 6610d4b7b6..87dd8fa4ad 100644--- a/apps/server/src/services/resources.ts+++ b/apps/server/src/services/resources.ts@@ -1,7 +1,7 @@import type { ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse } from "@triliumnext/commons";import { getLog, utils, ValidationError, ws } from "@triliumnext/core";import { t } from "i18next";-import { createHash } from "node:crypto";+import { createHash, type Hash } from "node:crypto";import fs from "node:fs";import path from "node:path";import { Readable, Transform } from "node:stream";@@ -58,7 +58,8 @@ const VERIFIED_FILE = ".verified.json";* downloads into a new directory. A file is written under a `.partial` name and renamed once its* size and SHA-256 match, then recorded in `.verified.json` with its size and modification time.* A file counts as installed only while it still matches that record, so one changed after its- * download is downloaded again, and an interrupted download keeps the files it finished.+ * download is downloaded again. An interrupted download keeps the files it finished and the+ * `.partial` of the one it was on, which the next download continues.*/export class ResourceManager {private readonly definitions = new Map<string, ResourceDefinition>();@@ -212,8 +213,12 @@ export class ResourceManager {private getInfo(definition: ResourceDefinition): ResourceInfo {const directory = this.getDirectory(definition);const verified = readVerifiedFiles(directory);- const downloaded = definition.files- .filter((file) => isVerified(verified, path.join(directory, file.path), file));+ const downloadedSize = definition.files.reduce((total, file) => {+ const destination = path.join(directory, file.path);+ return total + (isVerified(verified, destination, file)+ ? file.size+ : getPartialSize(destination, file));+ }, 0);return {id: definition.id,title: definition.title(),@@ -221,7 +226,7 @@ export class ResourceManager {icon: definition.icon,license: definition.license,size: sumSizes(definition.files),- downloadedSize: sumSizes(downloaded),+ downloadedSize,sources: [ ...new Set(definition.files.map(({ url }) => new URL(url).hostname)) ],installed: this.getPath(definition.id) !== null};@@ -249,20 +254,40 @@ export class ResourceManager {/*** Downloads `file` to `destination` through `<destination>.partial`, renaming it once its size and- * SHA-256 match. A failed or aborted download leaves no file behind.+ * SHA-256 match. A `.partial` left by an interrupted download is continued: its bytes are hashed+ * and the rest is asked for with a `Range` header, or the file starts over when the server sends+ * all of it. A failed download keeps its `.partial` while it is shorter than the file, so that the+ * next one continues it; a complete one that fails its hash is deleted.*/export async function downloadFile(file: ResourceFile, destination: string, signal: AbortSignal,onProgress: (bytes: number) => void) {const partial = `${destination}.partial`;const name = path.basename(file.path);try {- const response = await fetch(file.url, { signal });+ let hash = createHash("sha256");+ let received = getPartialSize(destination, file);+ if (received) {+ await hashFileInto(partial, hash);+ }+ const response = await fetch(file.url, {+ signal,+ headers: received ? { Range: `bytes=${received}-` } : undefined+ });if (!response.ok || !response.body) {throw new Error(t("resources.http-error", { name, status: response.status }));}+ if (received && response.status === 206) {+ const contentRange = response.headers.get("Content-Range") ?? "";+ if (!contentRange.startsWith(`bytes ${received}-`)) {+ await fs.promises.rm(partial, { force: true });+ throw new Error(t("resources.size-mismatch", { name }));+ }+ onProgress(received);+ } else {+ hash = createHash("sha256");+ received = 0;+ }- const hash = createHash("sha256");- let received = 0;const meter = new Transform({transform(chunk: Buffer, _encoding, callback) {received += chunk.length;@@ -276,7 +301,8 @@ export async function downloadFile(file: ResourceFile, destination: string, sign}});const body = Readable.fromWeb(response.body as NodeReadableStream<Uint8Array>);- await pipeline(body, meter, fs.createWriteStream(partial), { signal });+ const output = fs.createWriteStream(partial, { flags: received ? "a" : "w" });+ await pipeline(body, meter, output, { signal });if (received !== file.size) {throw new Error(t("resources.size-mismatch", { name }));@@ -286,7 +312,9 @@ export async function downloadFile(file: ResourceFile, destination: string, sign}await fs.promises.rename(partial, destination);} catch (e) {- await fs.promises.rm(partial, { force: true });+ if (getPartialSize(destination, file) === 0) {+ await fs.promises.rm(partial, { force: true });+ }throw e;}}@@ -313,6 +341,25 @@ function getContentHash(files: ResourceFile[]) {return hash.digest("hex").slice(0, CONTENT_HASH_LENGTH);}+/**+ * The bytes of `file` an interrupted download kept in `<destination>.partial`, or 0 when there is+ * none or it is already longer than the file can be.+ */+function getPartialSize(destination: string, file: ResourceFile) {+ try {+ const { size } = fs.statSync(`${destination}.partial`);+ return size < file.size ? size : 0;+ } catch {+ return 0;+ }+}++async function hashFileInto(filePath: string, hash: Hash) {+ for await (const chunk of fs.createReadStream(filePath)) {+ hash.update(chunk);+ }+}+function readVerifiedFiles(directory: string): VerifiedFiles {try {return JSON.parse(fs.readFileSync(path.join(directory, VERIFIED_FILE), "utf8"));@@ -352,7 +399,7 @@ async function hasContent(destination: string, file: ResourceFile) {return false;}const hash = createHash("sha256");- await pipeline(fs.createReadStream(destination), hash);+ await hashFileInto(destination, hash);return hash.digest("hex") === file.sha256;}