Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(resources): treat files copied from another device as in place
Elian Doranb408a32bed4f
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.jsonindex 417788a18d..fc412cf203 100644--- a/apps/client/src/translations/en/translation.json+++ b/apps/client/src/translations/en/translation.json@@ -4623,7 +4623,9 @@"cancel": "Cancel","not_enough_space_title": "Not enough free space","failed_title": "Download failed",- "license": "{{license}} license"+ "license": "{{license}} license",+ "verify": "Verify",+ "unverified": "{{size}} already on this device, checked instead of downloaded"},"transcription_options": {"recognition_title": "Speech recognition",@@ -4658,6 +4660,7 @@"unregistered_description": "A downloaded resource that no feature of this app uses.","unknown": "Unknown","unknown_description": "Not downloaded by Trilium.",- "delete_foreign_confirmation": "Delete {{name}}? No feature of this app uses it."+ "delete_foreign_confirmation": "Delete {{name}}? No feature of this app uses it.",+ "unverified": "Not checked yet"}}diff --git a/apps/client/src/widgets/dialogs/resource_download.spec.tsx b/apps/client/src/widgets/dialogs/resource_download.spec.tsxindex bb4b43728e..93cc5c0ade 100644--- a/apps/client/src/widgets/dialogs/resource_download.spec.tsx+++ b/apps/client/src/widgets/dialogs/resource_download.spec.tsx@@ -36,6 +36,7 @@ const STATUS: ResourcesStatusResponse = {license: "CC-BY-4.0",size: 700,downloadedSize: 200,+ unverifiedSize: 0,sources: [ "huggingface.co", "github.com" ],installed: false,directory: null@@ -164,6 +165,18 @@ describe("ResourceDownloadDialog", () => {expect(findButton("resource_download.retry")).toBeDefined();});+ it("offers to check files already on the device instead of downloading them", async () => {+ const copied = { ...STATUS.resources[0], downloadedSize: 0, unverifiedSize: 700 };+ await open({ ...STATUS, resources: [ copied ], freeBytes: 100 });++ expect(text(".resource-download-meta")).toContain(`resource_download.unverified:{"size":"700 B"}`);+ expect(text(".resource-download-item .resource-download-size")).toBe("0 B");+ expect(text(".resource-download-total .resource-download-size")).toBe("0 B");+ expect(container.querySelector(".admonition")).toBeNull();+ expect(findButton("resource_download.download")).toBeUndefined();+ expect(findButton("resource_download.verify")?.disabled).toBe(false);+ });+it("refuses to start a download that cannot fit in the free space", async () => {await open({ ...STATUS, freeBytes: 100 });@@ -230,7 +243,7 @@ function installed(status: ResourcesStatusResponse): ResourcesStatusResponse {return {...status,resources: status.resources.map((resource) => ({- ...resource, installed: true, downloadedSize: resource.size, directory: `/resources/${resource.id}`+ ...resource, installed: true, downloadedSize: resource.size, unverifiedSize: 0, directory: `/resources/${resource.id}`}))};}diff --git a/apps/client/src/widgets/dialogs/resource_download.tsx b/apps/client/src/widgets/dialogs/resource_download.tsxindex cf3e1cb02a..f7c6d5c08d 100644--- a/apps/client/src/widgets/dialogs/resource_download.tsx+++ b/apps/client/src/widgets/dialogs/resource_download.tsx@@ -97,6 +97,9 @@ export default function ResourceDownloadDialog() {const remaining = opts ? getRemainingSize(opts.status) : 0;const freeBytes = opts?.status.freeBytes ?? null;const lacksSpace = freeBytes !== null && freeBytes < remaining;+ const startText = isFailed+ ? t("resource_download.retry")+ : remaining === 0 ? t("resource_download.verify") : t("resource_download.download");return (<Modal@@ -113,8 +116,8 @@ export default function ResourceDownloadDialog() {<Button text={t("resource_download.cancel")} onClick={() => setShown(false)} />{!isDownloading && (<Button- text={isFailed ? t("resource_download.retry") : t("resource_download.download")}- icon="bx bx-download"+ text={startText}+ icon={remaining === 0 ? "bx bx-check-shield" : "bx bx-download"}kind="primary"disabled={lacksSpace}disabledTooltip={t("resource_download.not_enough_space_title")}@@ -210,7 +213,8 @@ interface ResourceItemProps {* to download of it, so that the rows add up to the download size. Once a download starts, the* icon shows whether the resource is done, downloading or waiting, as the setup's sync steps do.* The resource downloading carries a progress bar, and so does one an earlier download left- * partly fetched, filled as far as it got.+ * partly fetched, filled as far as it got. Files already in place but not yet checked, such as+ * ones copied from another device, are named in the source line and left out of the size.*/function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {const isActive = file !== undefined;@@ -227,6 +231,10 @@ function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {<span className="resource-download-license">{t("resource_download.license", { license: resource.license })}</span>+ {resource.unverifiedSize > 0 && <>+ {" · "}+ {t("resource_download.unverified", { size: formatSize(resource.unverifiedSize) })}+ </>}</small>{isActive && state && (<RowProgress@@ -244,7 +252,7 @@ function ResourceItem({ resource, unitOf, state, file }: ResourceItemProps) {)}</div><span className="tn-card-option-value resource-download-size">- {formatSize(resource.size - resource.downloadedSize, 2, unitOf)}+ {formatSize(getMissingSize(resource), 2, unitOf)}</span></CardSection>);@@ -289,5 +297,10 @@ function getRowIcon(resource: ResourceInfo, state: ResourceDownloadState | undeffunction getRemainingSize(status: ResourcesStatusResponse) {return status.resources.filter(({ installed }) => !installed)- .reduce((total, { size, downloadedSize }) => total + size - downloadedSize, 0);+ .reduce((total, resource) => total + getMissingSize(resource), 0);+}++/** The bytes of `resource` neither downloaded nor already in place waiting to be checked. */+function getMissingSize({ size, downloadedSize, unverifiedSize }: ResourceInfo) {+ return size - downloadedSize - unverifiedSize;}diff --git a/apps/client/src/widgets/type_widgets/options/resources.spec.tsx b/apps/client/src/widgets/type_widgets/options/resources.spec.tsxindex 9026d205f5..6094279cb5 100644--- a/apps/client/src/widgets/type_widgets/options/resources.spec.tsx+++ b/apps/client/src/widgets/type_widgets/options/resources.spec.tsx@@ -123,6 +123,18 @@ describe("ResourcesSettings", () => {.toEqual([ "2.00 GiB", "0.01 GiB", "2.00 GiB" ]);});+ it("lists files copied in but not yet checked, and counts them in the space used", async () => {+ mocks.resources = [ resource({ id: "test.copied", unverifiedSize: 6 * MIB }) ];+ await open();++ const rows = [ ...host.querySelectorAll(".resources-item") ];+ expect(rows.map((row) => row.querySelector(".tn-card-option-title")?.textContent))+ .toEqual([ "test.copied" ]);+ expect(rows[0].querySelector(".resources-status small")?.textContent)+ .toBe("resources_options.unverified");+ expect(host.querySelector(".resources-total .resources-size")?.textContent).toBe("6.00 MiB");+ });+it("opens the folder of an installed resource, on desktop only", async () => {await open();const buttons = () => [ ...host.querySelectorAll(".resources-item") ]@@ -257,6 +269,7 @@ function resource(overrides: Partial<ResourceInfo> & { id: string }): ResourceInlicense: "MIT",size: 10 * MIB,downloadedSize: 0,+ unverifiedSize: 0,sources: [ "example.com" ],installed: false,directory: null,diff --git a/apps/client/src/widgets/type_widgets/options/resources.tsx b/apps/client/src/widgets/type_widgets/options/resources.tsxindex 74509f5a9b..32428d8be6 100644--- a/apps/client/src/widgets/type_widgets/options/resources.tsx+++ b/apps/client/src/widgets/type_widgets/options/resources.tsx@@ -62,7 +62,7 @@ function useResources() {const [ status, setStatus ] = useState<ResourcesStatus>();const refresh = useCallback(() => {void server.get<ResourcesStatusResponse>("resources").then(({ resources, foreign, freeBytes }) => setStatus({- resources: resources.filter(({ downloadedSize }) => downloadedSize > 0),+ resources: resources.filter((resource) => getSizeOnDisk(resource) > 0),foreign: foreign ?? [],freeBytes}));@@ -88,7 +88,7 @@ interface ResourceRowsProps extends ResourcesStatus {/** The entries and the space they use, every size in one unit so that the column adds up. */function ResourceRows({ resources, foreign, freeBytes, onDeleted }: ResourceRowsProps) {- const used = resources.reduce((total, { downloadedSize }) => total + downloadedSize, 0)+ const used = resources.reduce((total, resource) => total + getSizeOnDisk(resource), 0)+ foreign.reduce((total, { size }) => total + size, 0);const unitOf = Math.max(used, ...resources.map(({ size }) => size));return (@@ -233,12 +233,20 @@ function EntryRow({ className, icon, title, children, size, status, onOpen, onDe);}-function getStatusText({ installed, size, downloadedSize }: ResourceInfo) {+function getStatusText({ installed, size, downloadedSize, unverifiedSize }: ResourceInfo) {if (installed) {return t("resources_options.installed");}+ if (unverifiedSize > 0) {+ return t("resources_options.unverified");+ }return t("resources_options.partially_downloaded", {downloaded: formatSize(downloadedSize, 2, size),size: formatSize(size)});}++/** The bytes of `resource` on disk: downloaded, kept from an interrupted download, or copied in. */+function getSizeOnDisk({ downloadedSize, unverifiedSize }: ResourceInfo) {+ return downloadedSize + unverifiedSize;+}diff --git a/apps/server/src/services/resources.spec.ts b/apps/server/src/services/resources.spec.tsindex 049a6c7afd..761a7afc8e 100644--- a/apps/server/src/services/resources.spec.ts+++ b/apps/server/src/services/resources.spec.ts@@ -121,6 +121,7 @@ describe("ResourceManager", () => {license: "MIT",size: MODEL.length + TOKENS.length,downloadedSize: MODEL.length + TOKENS.length,+ unverifiedSize: 0,sources: [ "127.0.0.1" ],installed: true,directory@@ -234,13 +235,19 @@ describe("ResourceManager", () => {manager.startDownload([ "test.model" ]);await waitForEnd();fs.rmSync(path.join(resourceDirectory(), ".verified.json"));+ fs.writeFileSync(path.join(resourceDirectory(), "tokens.txt"), "short");expect(manager.getPath("test.model")).toBeNull();+ expect((await manager.getStatus([ "test.model" ])).resources[0])+ .toMatchObject({ installed: false, downloadedSize: 0, unverifiedSize: MODEL.length });requested.length = 0;manager.startDownload([ "test.model" ]);expect((await waitForEnd()).status).toBe("succeeded");- expect(requested).toEqual([]);+ expect(requested).toEqual([ "/tokens.txt" ]);expect(manager.getPath("test.model")).toBe(resourceDirectory());+ expect((await manager.getStatus([ "test.model" ])).resources[0]).toMatchObject({+ installed: true, downloadedSize: MODEL.length + TOKENS.length, unverifiedSize: 0+ });});it("resumes a file after the connection drops, asking only for the bytes it is missing", async () => {diff --git a/apps/server/src/services/resources.ts b/apps/server/src/services/resources.tsindex 18a7283596..2c1f57b0ca 100644--- a/apps/server/src/services/resources.ts+++ b/apps/server/src/services/resources.ts@@ -301,12 +301,18 @@ export class ResourceManager {private getInfo(definition: ResourceDefinition): ResourceInfo {const directory = this.getDirectory(definition);const verified = readVerifiedFiles(directory);- const downloadedSize = definition.files.reduce((total, file) => {+ let downloadedSize = 0;+ let unverifiedSize = 0;+ for (const file of definition.files) {const destination = path.join(directory, file.path);- return total + (isVerified(verified, destination, file)- ? file.size- : getPartialSize(destination, file));- }, 0);+ if (isVerified(verified, destination, file)) {+ downloadedSize += file.size;+ } else if (hasSize(destination, file)) {+ unverifiedSize += file.size;+ } else {+ downloadedSize += getPartialSize(destination, file);+ }+ }const installedDirectory = this.getPath(definition.id);return {id: definition.id,@@ -316,6 +322,7 @@ export class ResourceManager {license: definition.license,size: sumSizes(definition.files),downloadedSize,+ unverifiedSize,sources: [ ...new Set(definition.files.map(({ url }) => new URL(url).hostname)) ],installed: installedDirectory !== null,directory: installedDirectory@@ -517,6 +524,15 @@ function isVerified(verified: VerifiedFiles, destination: string, file: Resource}/** Whether `destination` already holds `file`, as one put there by hand: checked by its hash. */+/** Whether `destination` is there with the size of `file`, whatever its content. */+function hasSize(destination: string, file: ResourceFile) {+ try {+ return fs.statSync(destination).size === file.size;+ } catch {+ return false;+ }+}+async function hasContent(destination: string, file: ResourceFile) {try {if ((await fs.promises.stat(destination)).size !== file.size) {diff --git a/packages/commons/src/lib/resources.ts b/packages/commons/src/lib/resources.tsindex 1eda3621a6..89b0f69f66 100644--- a/packages/commons/src/lib/resources.ts+++ b/packages/commons/src/lib/resources.ts@@ -17,6 +17,12 @@ export interface ResourceInfo {size: number;/** The bytes of its files already downloaded and verified, kept from an interrupted download. */downloadedSize: number;+ /**+ * The bytes of its files already in place with the right size but not yet checked against+ * their SHA-256, such as files copied from another device. The next download checks them+ * instead of downloading them.+ */+ unverifiedSize: number;/** The host names its files are downloaded from. */sources: string[];installed: boolean;