Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(resources): stop verifying on cancel and drop a copy that fails its hash
Elian Doran72aa9d3b68f1
diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.tsindex a969f905bb..a818c8591b 100644--- a/apps/server/src/services/resources.spec.ts+++ b/apps/server/src/services/resources.spec.ts@@ -272,6 +272,40 @@ describe("ResourceManager", () => {});});+ it("stops verifying files put in place by hand once the download is canceled", async () => {+ manager.register(defineModel());+ manager.startDownload([ "test.model" ]);+ await waitForEnd();+ fs.rmSync(path.join(resourceDirectory(), ".verified.json"));++ requested.length = 0;+ const { downloadId } = manager.startDownload([ "test.model" ]);+ manager.cancelDownload(downloadId);+ expect((await waitForEnd()).status).toBe("canceled");+ expect(requested).toEqual([]);+ expect(manager.getPath("test.model")).toBeNull();+ });++ it("deletes a file put in place by hand that fails its hash before downloading it again", async () => {+ manager.register(defineModel());+ manager.startDownload([ "test.model" ]);+ await waitForEnd();+ const tokens = path.join(resourceDirectory(), "tokens.txt");+ fs.writeFileSync(tokens, "x\ny\nz\n");+ fs.utimesSync(tokens, new Date(2020, 0, 1), new Date(2020, 0, 1));++ // The URL is not part of the directory's hash, so this manager finds the same files.+ const stalling = new ResourceManager(baseDir, (progress) => broadcasts.push(progress));+ stalling.register(defineModel({ tokensUrl: "/stall" }));+ expect((await stalling.getStatus([ "test.model" ])).resources[0].unverifiedSize)+ .toBe(TOKENS.length);+ const { downloadId } = stalling.startDownload([ "test.model" ]);+ await waitFor(() => stalled.length > 0);+ expect(fs.existsSync(tokens)).toBe(false);+ stalling.cancelDownload(downloadId);+ expect((await waitForEnd()).status).toBe("canceled");+ });+it("resumes a file after the connection drops, asking only for the bytes it is missing", async () => {manager.register(defineModel());dropModelAfter = 120_000;diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.tsindex 64c40c0846..115b255a72 100644--- a/apps/server/src/services/resources.ts+++ b/apps/server/src/services/resources.ts@@ -251,6 +251,7 @@ export class ResourceManager {const directory = this.getDirectory(definition);const verified = readVerifiedFiles(directory);for (const file of definition.files) {+ controller.signal.throwIfAborted();const destination = path.join(directory, file.path);if (isVerified(verified, destination, file)) {receive(file.size);@@ -259,7 +260,7 @@ export class ResourceManager {progress.file = path.basename(file.path);report(true);await fs.promises.mkdir(path.dirname(destination), { recursive: true });- if (await hasContent(destination, file)) {+ if (await hasContent(destination, file, controller.signal)) {receive(file.size);} else {await downloadFile(file, destination, controller.signal, (bytes) => {@@ -433,7 +434,7 @@ export async function downloadFile(file: ResourceFile, destination: string, signlet hash = createHash("sha256");let received = getPartialSize(destination, file);if (received) {- await hashFileInto(partial, hash);+ await hashFileInto(partial, hash, signal);}waitForBytes();const response = await fetch(file.url, {@@ -551,8 +552,8 @@ function getPartialSize(destination: string, file: ResourceFile) {}}-async function hashFileInto(filePath: string, hash: Hash) {- for await (const chunk of fs.createReadStream(filePath)) {+async function hashFileInto(filePath: string, hash: Hash, signal: AbortSignal) {+ for await (const chunk of fs.createReadStream(filePath, { signal })) {hash.update(chunk);}}@@ -596,7 +597,11 @@ function hasSize(destination: string, file: ResourceFile) {}}-async function hasContent(destination: string, file: ResourceFile) {+/**+ * Whether `destination` holds `file`. A file of the right size with other content is deleted, so+ * that downloading it again does not need room for two copies.+ */+async function hasContent(destination: string, file: ResourceFile, signal: AbortSignal) {try {if ((await fs.promises.stat(destination)).size !== file.size) {return false;@@ -605,8 +610,12 @@ async function hasContent(destination: string, file: ResourceFile) {return false;}const hash = createHash("sha256");- await hashFileInto(destination, hash);- return hash.digest("hex") === file.sha256;+ await hashFileInto(destination, hash, signal);+ if (hash.digest("hex") === file.sha256) {+ return true;+ }+ await fs.promises.rm(destination, { force: true });+ return false;}/** A snapshot of `progress`, which the running download goes on changing. */