Skip to content

Build your personal knowledge base with Trilium Notes

feat(resources): treat files copied from another device as in place

Elian Doranb408a32bed4f
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json
index 417788a18d..fc412cf203 100644
--- a/apps/client/src/translations/en/translation.json
+++ b/apps/client/src/translations/en/translation.json
@@ -4623,7 +4623,9 @@
"cancel": "Cancel",
"not_enough_space_title": "Not enough free space",
"failed_title": "Download failed",
- "license": "{{license}} license"
+ "license": "{{license}} license",
+ "verify": "Verify",
+ "unverified": "{{size}} already on this device, checked instead of downloaded"
},
"transcription_options": {
"recognition_title": "Speech recognition",
@@ -4658,6 +4660,7 @@
"unregistered_description": "A downloaded resource that no feature of this app uses.",
"unknown": "Unknown",
"unknown_description": "Not downloaded by Trilium.",
- "delete_foreign_confirmation": "Delete {{name}}? No feature of this app uses it."
+ "delete_foreign_confirmation": "Delete {{name}}? No feature of this app uses it.",
+ "unverified": "Not checked yet"
}
}
diff --git a/apps/client/src/widgets/dialogs/resource_download.spec.tsx b/apps/client/src/widgets/dialogs/resource_download.spec.tsx
index bb4b43728e..93cc5c0ade 100644
--- a/apps/client/src/widgets/dialogs/resource_download.spec.tsx
+++ b/apps/client/src/widgets/dialogs/resource_download.spec.tsx
@@ -36,6 +36,7 @@ const STATUS: ResourcesStatusResponse = {
license: "CC-BY-4.0",
size: 700,
downloadedSize: 200,
+ unverifiedSize: 0,
sources: [ "huggingface.co", "github.com" ],
installed: false,
directory: null
@@ -164,6 +165,18 @@ describe("ResourceDownloadDialog", () => {
expect(findButton("resource_download.retry")).toBeDefined();
});
+ it("offers to check files already on the device instead of downloading them", async () => {
+ const copied = { ...STATUS.resources[0], downloadedSize: 0, unverifiedSize: 700 };
+ await open({ ...STATUS, resources: [ copied ], freeBytes: 100 });
+
+ expect(text(".resource-download-meta")).toContain(`resource_download.unverified:{"size":"700 B"}`);
+ expect(text(".resource-download-item .resource-download-size")).toBe("0 B");
+ expect(text(".resource-download-total .resource-download-size")).toBe("0 B");
+ expect(container.querySelector(".admonition")).toBeNull();
+ expect(findButton("resource_download.download")).toBeUndefined();
+ expect(findButton("resource_download.verify")?.disabled).toBe(false);
+ });
+
it("refuses to start a download that cannot fit in the free space", async () => {
await open({ ...STATUS, freeBytes: 100 });
@@ -230,7 +243,7 @@ function installed(status: ResourcesStatusResponse): ResourcesStatusResponse {
return {
...status,
resources: status.resources.map((resource) => ({
- ...resource, installed: true, downloadedSize: resource.size, directory: `/resources/${resource.id}`
+ ...resource, installed: true, downloadedSize: resource.size, unverifiedSize: 0, directory: `/resources/${resource.id}`
}))
};
}
diff --git a/apps/client/src/widgets/dialogs/resource_download.tsx b/apps/client/src/widgets/dialogs/resource_download.tsx
index cf3e1cb02a..f7c6d5c08d 100644
--- a/apps/client/src/widgets/dialogs/resource_download.tsx
+++ b/apps/client/src/widgets/dialogs/resource_download.tsx
@@ -97,6 +97,9 @@ export default function ResourceDownloadDialog() {
const remaining = opts ? getRemainingSize(opts.status) : 0;
const freeBytes = opts?.status.freeBytes ?? null;
const lacksSpace = freeBytes !== null && freeBytes < remaining;
+ const startText = isFailed
+ ? t("resource_download.retry")
+ : remaining === 0 ? t("resource_download.verify") : t("resource_download.download");
return (
<Modal
@@ -113,8 +116,8 @@ export default function ResourceDownloadDialog() {
<Button text={t("resource_download.cancel")} onClick={() => setShown(false)} />
{!isDownloading && (
<Button
- text={isFailed ? t("resource_download.retry") : t("resource_download.download")}
- icon="bx bx-download"
+ text={startText}
+ icon={remaining === 0 ? "bx bx-check-shield" : "bx bx-download"}
kind="primary"
disabled={lacksSpace}
disabledTooltip={t("resource_download.not_enough_space_title")}
@@ -210,7 +213,8 @@ interface ResourceItemProps {
* to download of it, so that the rows add up to the download size. Once a download starts, the
* icon shows whether the resource is done, downloading or waiting, as the setup's sync steps do.
* The resource downloading carries a progress bar, and so does one an earlier download left
- * partly fetched, filled as far as it got.
+ * partly fetched, filled as far as it got. Files already in place but not yet checked, such as
+ * ones copied from another device, are named in the source line and left out of the size.
*/
function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {
const isActive = file !== undefined;
@@ -227,6 +231,10 @@ function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {
<span className="resource-download-license">
{t("resource_download.license", { license: resource.license })}
</span>
+ {resource.unverifiedSize > 0 && <>
+ {" · "}
+ {t("resource_download.unverified", { size: formatSize(resource.unverifiedSize) })}
+ </>}
</small>
{isActive && state && (
<RowProgress
@@ -244,7 +252,7 @@ function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {
)}
</div>
<span className="tn-card-option-value resource-download-size">
- {formatSize(resource.size - resource.downloadedSize, 2, unitOf)}
+ {formatSize(getMissingSize(resource), 2, unitOf)}
</span>
</CardSection>
);
@@ -289,5 +297,10 @@ function getRowIcon(resource: ResourceInfo, state: ResourceDownloadState | undef
function getRemainingSize(status: ResourcesStatusResponse) {
return status.resources
.filter(({ installed }) => !installed)
- .reduce((total, { size, downloadedSize }) => total + size - downloadedSize, 0);
+ .reduce((total, resource) => total + getMissingSize(resource), 0);
+}
+
+/** The bytes of `resource` neither downloaded nor already in place waiting to be checked. */
+function getMissingSize({ size, downloadedSize, unverifiedSize }: ResourceInfo) {
+ return size - downloadedSize - unverifiedSize;
}
diff --git a/apps/client/src/widgets/type_widgets/options/resources.spec.tsx b/apps/client/src/widgets/type_widgets/options/resources.spec.tsx
index 9026d205f5..6094279cb5 100644
--- a/apps/client/src/widgets/type_widgets/options/resources.spec.tsx
+++ b/apps/client/src/widgets/type_widgets/options/resources.spec.tsx
@@ -123,6 +123,18 @@ describe("ResourcesSettings", () => {
.toEqual([ "2.00 GiB", "0.01 GiB", "2.00 GiB" ]);
});
+ it("lists files copied in but not yet checked, and counts them in the space used", async () => {
+ mocks.resources = [ resource({ id: "test.copied", unverifiedSize: 6 * MIB }) ];
+ await open();
+
+ const rows = [ ...host.querySelectorAll(".resources-item") ];
+ expect(rows.map((row) => row.querySelector(".tn-card-option-title")?.textContent))
+ .toEqual([ "test.copied" ]);
+ expect(rows[0].querySelector(".resources-status small")?.textContent)
+ .toBe("resources_options.unverified");
+ expect(host.querySelector(".resources-total .resources-size")?.textContent).toBe("6.00 MiB");
+ });
+
it("opens the folder of an installed resource, on desktop only", async () => {
await open();
const buttons = () => [ ...host.querySelectorAll(".resources-item") ]
@@ -257,6 +269,7 @@ function resource(overrides: Partial<ResourceInfo> & { id: string }): ResourceIn
license: "MIT",
size: 10 * MIB,
downloadedSize: 0,
+ unverifiedSize: 0,
sources: [ "example.com" ],
installed: false,
directory: null,
diff --git a/apps/client/src/widgets/type_widgets/options/resources.tsx b/apps/client/src/widgets/type_widgets/options/resources.tsx
index 74509f5a9b..32428d8be6 100644
--- a/apps/client/src/widgets/type_widgets/options/resources.tsx
+++ b/apps/client/src/widgets/type_widgets/options/resources.tsx
@@ -62,7 +62,7 @@ function useResources() {
const [ status, setStatus ] = useState<ResourcesStatus>();
const refresh = useCallback(() => {
void server.get<ResourcesStatusResponse>("resources").then(({ resources, foreign, freeBytes }) => setStatus({
- resources: resources.filter(({ downloadedSize }) => downloadedSize > 0),
+ resources: resources.filter((resource) => getSizeOnDisk(resource) > 0),
foreign: foreign ?? [],
freeBytes
}));
@@ -88,7 +88,7 @@ interface ResourceRowsProps extends ResourcesStatus {
/** The entries and the space they use, every size in one unit so that the column adds up. */
function ResourceRows({ resources, foreign, freeBytes, onDeleted }: ResourceRowsProps) {
- const used = resources.reduce((total, { downloadedSize }) => total + downloadedSize, 0)
+ const used = resources.reduce((total, resource) => total + getSizeOnDisk(resource), 0)
+ foreign.reduce((total, { size }) => total + size, 0);
const unitOf = Math.max(used, ...resources.map(({ size }) => size));
return (
@@ -233,12 +233,20 @@ function EntryRow({ className, icon, title, children, size, status, onOpen, onDe
);
}
-function getStatusText({ installed, size, downloadedSize }: ResourceInfo) {
+function getStatusText({ installed, size, downloadedSize, unverifiedSize }: ResourceInfo) {
if (installed) {
return t("resources_options.installed");
}
+ if (unverifiedSize > 0) {
+ return t("resources_options.unverified");
+ }
return t("resources_options.partially_downloaded", {
downloaded: formatSize(downloadedSize, 2, size),
size: formatSize(size)
});
}
+
+/** The bytes of `resource` on disk: downloaded, kept from an interrupted download, or copied in. */
+function getSizeOnDisk({ downloadedSize, unverifiedSize }: ResourceInfo) {
+ return downloadedSize + unverifiedSize;
+}
diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.ts
index 049a6c7afd..761a7afc8e 100644
--- a/apps/server/src/services/resources.spec.ts
+++ b/apps/server/src/services/resources.spec.ts
@@ -121,6 +121,7 @@ describe("ResourceManager", () => {
license: "MIT",
size: MODEL.length + TOKENS.length,
downloadedSize: MODEL.length + TOKENS.length,
+ unverifiedSize: 0,
sources: [ "127.0.0.1" ],
installed: true,
directory
@@ -234,13 +235,19 @@ describe("ResourceManager", () => {
manager.startDownload([ "test.model" ]);
await waitForEnd();
fs.rmSync(path.join(resourceDirectory(), ".verified.json"));
+ fs.writeFileSync(path.join(resourceDirectory(), "tokens.txt"), "short");
expect(manager.getPath("test.model")).toBeNull();
+ expect((await manager.getStatus([ "test.model" ])).resources[0])
+ .toMatchObject({ installed: false, downloadedSize: 0, unverifiedSize: MODEL.length });
requested.length = 0;
manager.startDownload([ "test.model" ]);
expect((await waitForEnd()).status).toBe("succeeded");
- expect(requested).toEqual([]);
+ expect(requested).toEqual([ "/tokens.txt" ]);
expect(manager.getPath("test.model")).toBe(resourceDirectory());
+ expect((await manager.getStatus([ "test.model" ])).resources[0]).toMatchObject({
+ installed: true, downloadedSize: MODEL.length + TOKENS.length, unverifiedSize: 0
+ });
});
it("resumes a file after the connection drops, asking only for the bytes it is missing", async () => {
diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.ts
index 18a7283596..2c1f57b0ca 100644
--- a/apps/server/src/services/resources.ts
+++ b/apps/server/src/services/resources.ts
@@ -301,12 +301,18 @@ export class ResourceManager {
private getInfo(definition: ResourceDefinition): ResourceInfo {
const directory = this.getDirectory(definition);
const verified = readVerifiedFiles(directory);
- const downloadedSize = definition.files.reduce((total, file) => {
+ let downloadedSize = 0;
+ let unverifiedSize = 0;
+ for (const file of definition.files) {
const destination = path.join(directory, file.path);
- return total + (isVerified(verified, destination, file)
- ? file.size
- : getPartialSize(destination, file));
- }, 0);
+ if (isVerified(verified, destination, file)) {
+ downloadedSize += file.size;
+ } else if (hasSize(destination, file)) {
+ unverifiedSize += file.size;
+ } else {
+ downloadedSize += getPartialSize(destination, file);
+ }
+ }
const installedDirectory = this.getPath(definition.id);
return {
id: definition.id,
@@ -316,6 +322,7 @@ export class ResourceManager {
license: definition.license,
size: sumSizes(definition.files),
downloadedSize,
+ unverifiedSize,
sources: [ ...new Set(definition.files.map(({ url }) => new URL(url).hostname)) ],
installed: installedDirectory !== null,
directory: installedDirectory
@@ -517,6 +524,15 @@ function isVerified(verified: VerifiedFiles, destination: string, file: Resource
}
/** Whether `destination` already holds `file`, as one put there by hand: checked by its hash. */
+/** Whether `destination` is there with the size of `file`, whatever its content. */
+function hasSize(destination: string, file: ResourceFile) {
+ try {
+ return fs.statSync(destination).size === file.size;
+ } catch {
+ return false;
+ }
+}
+
async function hasContent(destination: string, file: ResourceFile) {
try {
if ((await fs.promises.stat(destination)).size !== file.size) {
diff --git a/packages/commons/src/lib/resources.ts b/packages/commons/src/lib/resources.ts
index 1eda3621a6..89b0f69f66 100644
--- a/packages/commons/src/lib/resources.ts
+++ b/packages/commons/src/lib/resources.ts
@@ -17,6 +17,12 @@ export interface ResourceInfo {
size: number;
/** The bytes of its files already downloaded and verified, kept from an interrupted download. */
downloadedSize: number;
+ /**
+ * The bytes of its files already in place with the right size but not yet checked against
+ * their SHA-256, such as files copied from another device. The next download checks them
+ * instead of downloading them.
+ */
+ unverifiedSize: number;
/** The host names its files are downloaded from. */
sources: string[];
installed: boolean;