Skip to content

Build your personal knowledge base with Trilium Notes

feat(resources): resume an interrupted download from the bytes it kept

Elian Dorancff77e3bfd7b
diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.ts
index 675a6d44e7..732fa1a5c2 100644
--- a/apps/server/src/services/resources.spec.ts
+++ b/apps/server/src/services/resources.spec.ts
@@ -15,6 +15,10 @@ const TOKENS = Buffer.from("a\nb\nc\n");
let server: http.Server;
let baseUrl = "";
const requested: string[] = [];
+/** The `Range` header of each request, in order. */
+const ranges: (string | undefined)[] = [];
+/** Makes the next whole-file request for `/model.bin` drop the connection after this many bytes. */
+let dropModelAfter: number | null = null;
/** Requests to `/stall`, answered with headers and no body until the test ends. */
let stalled: http.ServerResponse[] = [];
@@ -27,7 +31,13 @@ let manager: ResourceManager;
beforeAll(async () => {
server = http.createServer((req, res) => {
requested.push(req.url ?? "");
+ ranges.push(req.headers.range);
if (req.url === "/model.bin") {
+ serveModel(req, res);
+ } else if (req.url === "/model-no-range.bin") {
+ res.end(MODEL);
+ } else if (req.url === "/model-wrong-range.bin") {
+ res.writeHead(206, { "Content-Range": `bytes 0-${MODEL.length - 1}/${MODEL.length}` });
res.end(MODEL);
} else if (req.url === "/tokens.txt") {
res.end(TOKENS);
@@ -49,6 +59,8 @@ beforeEach(() => {
baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-resources-"));
broadcasts = [];
requested.length = 0;
+ ranges.length = 0;
+ dropModelAfter = null;
manager = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));
});
@@ -133,7 +145,7 @@ describe("ResourceManager", () => {
expect(manager.getPath("test.short")).toBeNull();
});
- it("joins a running download, and cancels it without leaving a partial file", async () => {
+ it("joins a running download, and cancels it without leaving an empty partial file", async () => {
manager.register(defineModel({ tokensUrl: "/stall" }));
const first = manager.startDownload([ "test.model" ]);
@@ -191,6 +203,79 @@ describe("ResourceManager", () => {
expect(manager.getPath("test.model")).toBe(resourceDirectory());
});
+ it("resumes a file after the connection drops, asking only for the bytes it is missing", async () => {
+ manager.register(defineModel());
+ dropModelAfter = 120_000;
+ manager.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).status).toBe("failed");
+
+ const partial = path.join(resourceDirectory(), "nested", "model.bin.partial");
+ const kept = fs.statSync(partial).size;
+ expect(kept).toBeGreaterThan(0);
+ expect(kept).toBeLessThanOrEqual(120_000);
+ expect((await manager.getStatus([ "test.model" ])).resources[0].downloadedSize).toBe(kept);
+
+ ranges.length = 0;
+ manager.startDownload([ "test.model" ]);
+ expect(await waitForEnd()).toMatchObject({
+ status: "succeeded", receivedBytes: MODEL.length + TOKENS.length
+ });
+ expect(ranges[0]).toBe(`bytes=${kept}-`);
+ expect(fs.readFileSync(path.join(resourceDirectory(), "nested", "model.bin")).equals(MODEL))
+ .toBe(true);
+ });
+
+ it("starts over when the server sends the whole file instead of the rest", async () => {
+ manager.register(defineModel());
+ dropModelAfter = 120_000;
+ manager.startDownload([ "test.model" ]);
+ await waitForEnd();
+
+ const resumed = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));
+ resumed.register(defineModel({ modelUrl: "/model-no-range.bin" }));
+ ranges.length = 0;
+ resumed.startDownload([ "test.model" ]);
+ expect(await waitForEnd()).toMatchObject({
+ status: "succeeded", receivedBytes: MODEL.length + TOKENS.length
+ });
+ expect(ranges[0]?.startsWith("bytes=")).toBe(true);
+ expect(fs.readFileSync(path.join(resourceDirectory(), "nested", "model.bin")).equals(MODEL))
+ .toBe(true);
+ });
+
+ it("discards the kept bytes when the server answers with another range than asked", async () => {
+ manager.register(defineModel());
+ dropModelAfter = 120_000;
+ manager.startDownload([ "test.model" ]);
+ await waitForEnd();
+ const partial = path.join(resourceDirectory(), "nested", "model.bin.partial");
+ expect(fs.existsSync(partial)).toBe(true);
+
+ const resumed = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));
+ resumed.register(defineModel({ modelUrl: "/model-wrong-range.bin" }));
+ resumed.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).status).toBe("failed");
+ expect(fs.existsSync(partial)).toBe(false);
+ });
+
+ it("discards kept bytes that turn out wrong, and downloads the whole file on the next try", async () => {
+ manager.register(defineModel());
+ dropModelAfter = 120_000;
+ manager.startDownload([ "test.model" ]);
+ await waitForEnd();
+ const partial = path.join(resourceDirectory(), "nested", "model.bin.partial");
+ fs.writeFileSync(partial, Buffer.alloc(fs.statSync(partial).size, "x"));
+
+ manager.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).status).toBe("failed");
+ expect(fs.existsSync(partial)).toBe(false);
+
+ ranges.length = 0;
+ manager.startDownload([ "test.model" ]);
+ expect((await waitForEnd()).status).toBe("succeeded");
+ expect(ranges[0]).toBeUndefined();
+ });
+
it("prefers a local copy, and refuses invalid or unknown resources", () => {
manager.register({ ...defineModel(), findLocalCopy: () => "/hand/installed" });
expect(manager.getPath("test.model")).toBe("/hand/installed");
@@ -204,8 +289,8 @@ describe("ResourceManager", () => {
});
});
-function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash = sha256(TOKENS),
- tokensSize = TOKENS.length } = {}): ResourceDefinition {
+function defineModel({ id = "test.model", modelUrl = "/model.bin", tokensUrl = "/tokens.txt",
+ tokensHash = sha256(TOKENS), tokensSize = TOKENS.length } = {}): ResourceDefinition {
return {
id,
title: () => "Test model",
@@ -213,12 +298,32 @@ function defineModel({ id = "test.model", tokensUrl = "/tokens.txt", tokensHash
icon: "bx bx-test-tube",
license: "MIT",
files: [
- file("nested/model.bin", MODEL),
+ { ...file("nested/model.bin", MODEL), url: `${baseUrl}${modelUrl}` },
{ path: "tokens.txt", url: `${baseUrl}${tokensUrl}`, size: tokensSize, sha256: tokensHash }
]
};
}
+/**
+ * Serves the model, from the offset a `Range: bytes=<n>-` header asks for when there is one, or
+ * drops the connection after {@link dropModelAfter} bytes once when that is set.
+ */
+function serveModel(req: http.IncomingMessage, res: http.ServerResponse) {
+ const range = req.headers.range;
+ if (range?.startsWith("bytes=")) {
+ const start = Number(range.slice("bytes=".length, range.indexOf("-")));
+ res.writeHead(206, { "Content-Range": `bytes ${start}-${MODEL.length - 1}/${MODEL.length}` });
+ res.end(MODEL.subarray(start));
+ } else if (dropModelAfter !== null) {
+ const length = dropModelAfter;
+ dropModelAfter = null;
+ res.writeHead(200, { "Content-Length": MODEL.length });
+ res.write(MODEL.subarray(0, length), () => setTimeout(() => res.destroy(), 20));
+ } else {
+ res.end(MODEL);
+ }
+}
+
/** The directory the resource `id` was downloaded into: the only one named after it. */
function resourceDirectory(id = "test.model") {
const directories = fs.readdirSync(baseDir).filter((name) => name.startsWith(`${id}-`));
diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.ts
index 6610d4b7b6..87dd8fa4ad 100644
--- a/apps/server/src/services/resources.ts
+++ b/apps/server/src/services/resources.ts
@@ -1,7 +1,7 @@
import type { ResourceDownloadProgress, ResourceInfo, ResourcesStatusResponse } from "@triliumnext/commons";
import { getLog, utils, ValidationError, ws } from "@triliumnext/core";
import { t } from "i18next";
-import { createHash } from "node:crypto";
+import { createHash, type Hash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { Readable, Transform } from "node:stream";
@@ -58,7 +58,8 @@ const VERIFIED_FILE = ".verified.json";
* downloads into a new directory. A file is written under a `.partial` name and renamed once its
* size and SHA-256 match, then recorded in `.verified.json` with its size and modification time.
* A file counts as installed only while it still matches that record, so one changed after its
- * download is downloaded again, and an interrupted download keeps the files it finished.
+ * download is downloaded again. An interrupted download keeps the files it finished and the
+ * `.partial` of the one it was on, which the next download continues.
*/
export class ResourceManager {
private readonly definitions = new Map<string, ResourceDefinition>();
@@ -212,8 +213,12 @@ export class ResourceManager {
private getInfo(definition: ResourceDefinition): ResourceInfo {
const directory = this.getDirectory(definition);
const verified = readVerifiedFiles(directory);
- const downloaded = definition.files
- .filter((file) => isVerified(verified, path.join(directory, file.path), file));
+ const downloadedSize = definition.files.reduce((total, file) => {
+ const destination = path.join(directory, file.path);
+ return total + (isVerified(verified, destination, file)
+ ? file.size
+ : getPartialSize(destination, file));
+ }, 0);
return {
id: definition.id,
title: definition.title(),
@@ -221,7 +226,7 @@ export class ResourceManager {
icon: definition.icon,
license: definition.license,
size: sumSizes(definition.files),
- downloadedSize: sumSizes(downloaded),
+ downloadedSize,
sources: [ ...new Set(definition.files.map(({ url }) => new URL(url).hostname)) ],
installed: this.getPath(definition.id) !== null
};
@@ -249,20 +254,40 @@ export class ResourceManager {
/**
* Downloads `file` to `destination` through `<destination>.partial`, renaming it once its size and
- * SHA-256 match. A failed or aborted download leaves no file behind.
+ * SHA-256 match. A `.partial` left by an interrupted download is continued: its bytes are hashed
+ * and the rest is asked for with a `Range` header, or the file starts over when the server sends
+ * all of it. A failed download keeps its `.partial` while it is shorter than the file, so that the
+ * next one continues it; a complete one that fails its hash is deleted.
*/
export async function downloadFile(file: ResourceFile, destination: string, signal: AbortSignal,
onProgress: (bytes: number) => void) {
const partial = `${destination}.partial`;
const name = path.basename(file.path);
try {
- const response = await fetch(file.url, { signal });
+ let hash = createHash("sha256");
+ let received = getPartialSize(destination, file);
+ if (received) {
+ await hashFileInto(partial, hash);
+ }
+ const response = await fetch(file.url, {
+ signal,
+ headers: received ? { Range: `bytes=${received}-` } : undefined
+ });
if (!response.ok || !response.body) {
throw new Error(t("resources.http-error", { name, status: response.status }));
}
+ if (received && response.status === 206) {
+ const contentRange = response.headers.get("Content-Range") ?? "";
+ if (!contentRange.startsWith(`bytes ${received}-`)) {
+ await fs.promises.rm(partial, { force: true });
+ throw new Error(t("resources.size-mismatch", { name }));
+ }
+ onProgress(received);
+ } else {
+ hash = createHash("sha256");
+ received = 0;
+ }
- const hash = createHash("sha256");
- let received = 0;
const meter = new Transform({
transform(chunk: Buffer, _encoding, callback) {
received += chunk.length;
@@ -276,7 +301,8 @@ export async function downloadFile(file: ResourceFile, destination: string, sign
}
});
const body = Readable.fromWeb(response.body as NodeReadableStream<Uint8Array>);
- await pipeline(body, meter, fs.createWriteStream(partial), { signal });
+ const output = fs.createWriteStream(partial, { flags: received ? "a" : "w" });
+ await pipeline(body, meter, output, { signal });
if (received !== file.size) {
throw new Error(t("resources.size-mismatch", { name }));
@@ -286,7 +312,9 @@ export async function downloadFile(file: ResourceFile, destination: string, sign
}
await fs.promises.rename(partial, destination);
} catch (e) {
- await fs.promises.rm(partial, { force: true });
+ if (getPartialSize(destination, file) === 0) {
+ await fs.promises.rm(partial, { force: true });
+ }
throw e;
}
}
@@ -313,6 +341,25 @@ function getContentHash(files: ResourceFile[]) {
return hash.digest("hex").slice(0, CONTENT_HASH_LENGTH);
}
+/**
+ * The bytes of `file` an interrupted download kept in `<destination>.partial`, or 0 when there is
+ * none or it is already longer than the file can be.
+ */
+function getPartialSize(destination: string, file: ResourceFile) {
+ try {
+ const { size } = fs.statSync(`${destination}.partial`);
+ return size < file.size ? size : 0;
+ } catch {
+ return 0;
+ }
+}
+
+async function hashFileInto(filePath: string, hash: Hash) {
+ for await (const chunk of fs.createReadStream(filePath)) {
+ hash.update(chunk);
+ }
+}
+
function readVerifiedFiles(directory: string): VerifiedFiles {
try {
return JSON.parse(fs.readFileSync(path.join(directory, VERIFIED_FILE), "utf8"));
@@ -352,7 +399,7 @@ async function hasContent(destination: string, file: ResourceFile) {
return false;
}
const hash = createHash("sha256");
- await pipeline(fs.createReadStream(destination), hash);
+ await hashFileInto(destination, hash);
return hash.digest("hex") === file.sha256;
}