A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat(mobile): add the offline download sync engine (#3175)
Mohamed Bassema18fcb7c0f69
diff --git a/apps/mobile/lib/offlineSyncEngine.test.ts b/apps/mobile/lib/offlineSyncEngine.test.tsnew file mode 100644index 00000000..07392463--- /dev/null+++ b/apps/mobile/lib/offlineSyncEngine.test.ts@@ -0,0 +1,384 @@+import { describe, expect, it } from "vitest";++import type {+ PreparedCopy,+ StoredCopy,+ SyncCandidate,+ SyncDeps,+} from "./offlineSyncEngine";+import {+ ContentUnavailable,+ planRoomFor,+ selectCandidates,+ synchronizeOfflineCopies,+ SyncInterrupted,+} from "./offlineSyncEngine";+import { assetBookmark, linkBookmark, textBookmark } from "./testing/bookmarks";++const NOW = 1_000_000_000;++function candidate(bookmarkId: string, ready = true): SyncCandidate {+ return { bookmarkId, ready };+}++function copy(+ bookmarkId: string,+ overrides: Partial<StoredCopy> = {},+): StoredCopy {+ return {+ bookmarkId,+ source: "auto",+ byteSize: 10,+ savedAt: NOW - 1000,+ ...overrides,+ };+}++function harness({+ stored = [],+ budgetBytes = 1000,+ sizes = {},+ failing = new Set<string>(),+ checkpoint = async () => undefined,+}: {+ stored?: StoredCopy[];+ budgetBytes?: number;+ sizes?: Record<string, number>;+ failing?: Set<string>;+ checkpoint?: () => Promise<void>;+} = {}) {+ const library = new Map(stored.map((item) => [item.bookmarkId, item]));+ const downloads: string[] = [];+ const excluded = new Set<string>();+ const deps: SyncDeps = {+ budgetBytes,+ knownSizes: new Map(),+ listStored: () => [...library.values()],+ isExcluded: (bookmarkId) => excluded.has(bookmarkId),+ checkpoint,+ remove: (bookmarkId) => {+ library.delete(bookmarkId);+ },+ prepare: async (next): Promise<PreparedCopy> => {+ downloads.push(next.bookmarkId);+ if (failing.has(next.bookmarkId)) {+ throw new Error("download failed");+ }+ const byteSize = sizes[next.bookmarkId] ?? 10;+ return {+ byteSize,+ save: () => {+ library.set(next.bookmarkId, {+ bookmarkId: next.bookmarkId,+ source: "auto",+ byteSize,+ savedAt: NOW,+ });+ },+ };+ },+ };+ return {+ deps,+ downloads,+ excluded,+ ids: () => [...library.keys()].sort(),+ };+}++describe("selectCandidates", () => {+ it("puts list articles first, newest first, without duplicates", () => {+ const result = selectCandidates(+ [+ linkBookmark("list-old", { createdAt: 1 }),+ linkBookmark("both", { createdAt: 5 }),+ ],+ [+ linkBookmark("recent", { createdAt: 9 }),+ linkBookmark("both", { createdAt: 5 }),+ ],+ [],+ );+ expect(result.map((item) => item.bookmarkId)).toEqual([+ "both",+ "list-old",+ "recent",+ ]);+ });++ it("drops exclusions and assets, and marks pending crawls", () => {+ // Older servers report neither a crawl status nor a crawl time.+ const link = linkBookmark("uncrawled", { createdAt: 3 });+ const uncrawled = {+ ...link,+ content: {+ ...link.content,+ crawlStatus: undefined,+ crawledAt: undefined,+ },+ };+ const result = selectCandidates(+ [],+ [+ linkBookmark("excluded", { createdAt: 6 }),+ assetBookmark("pdf", 5),+ linkBookmark("broken", { createdAt: 4, crawlStatus: "failure" }),+ linkBookmark("pending", { createdAt: 3, crawlStatus: "pending" }),+ uncrawled,+ textBookmark("note", 2),+ ],+ ["excluded"],+ );+ expect(result).toEqual([+ // A failed crawl may have kept an earlier crawl's content.+ { bookmarkId: "broken", ready: true },+ { bookmarkId: "pending", ready: false },+ { bookmarkId: "uncrawled", ready: false },+ { bookmarkId: "note", ready: true },+ ]);+ });+});++describe("planRoomFor", () => {+ const ranks = new Map([+ ["a", 0],+ ["b", 1],+ ["c", 2],+ ]);+ const priority = (id: string) => ranks.get(id) ?? Number.POSITIVE_INFINITY;++ it("evicts unwanted copies first, then the lowest priority", () => {+ const stored = [+ copy("b"),+ copy("c"),+ copy("gone"),+ copy("mine", { source: "manual" }),+ ];+ expect(+ planRoomFor(+ { bookmarkId: "a", byteSize: 10, rank: 0 },+ stored,+ priority,+ 30,+ ),+ ).toEqual(["gone", "c"]);+ });++ it("evicts nothing when even evicting everything allowed is not enough", () => {+ const stored = [copy("a"), copy("c"), copy("mine", { source: "manual" })];+ expect(+ planRoomFor(+ { bookmarkId: "b", byteSize: 25, rank: 1 },+ stored,+ priority,+ 30,+ ),+ ).toBeUndefined();+ });+});++describe("synchronizeOfflineCopies", () => {+ it("downloads only what is missing", async () => {+ const h = harness({+ stored: [copy("stored"), copy("mine", { source: "manual" })],+ });+ const result = await synchronizeOfflineCopies(+ [candidate("stored"), candidate("mine"), candidate("new")],+ h.deps,+ );+ expect(h.downloads).toEqual(["new"]);+ expect(result).toMatchObject({ total: 3, ready: 3, failed: 0 });+ });++ it("skips an article without reader content until the next pass", async () => {+ const h = harness();+ const prepare = h.deps.prepare;+ h.deps.prepare = async (next) => {+ if (next.bookmarkId === "empty") {+ h.downloads.push(next.bookmarkId);+ throw new ContentUnavailable();+ }+ return prepare(next);+ };+ const result = await synchronizeOfflineCopies(+ [candidate("empty"), candidate("a")],+ h.deps,+ );+ expect(result).toMatchObject({ total: 1, ready: 1, failed: 0 });++ await synchronizeOfflineCopies([candidate("empty")], h.deps);+ expect(h.downloads).toEqual(["empty", "a", "empty"]);+ });++ it("rotates out articles that left the selection", async () => {+ const h = harness({+ stored: [copy("old"), copy("mine", { source: "manual" })],+ });+ await synchronizeOfflineCopies([candidate("new")], h.deps);+ expect(h.ids()).toEqual(["mine", "new"]);+ });++ it("keeps one old article per failed download, newest first", async () => {+ const h = harness({+ stored: [+ copy("older", { savedAt: NOW - 2000 }),+ copy("old", { savedAt: NOW - 1000 }),+ ],+ failing: new Set(["new"]),+ });+ await synchronizeOfflineCopies(+ [candidate("new"), candidate("pending", false)],+ h.deps,+ );+ expect(h.ids()).toEqual(["old", "older"]);++ const next = harness({+ stored: [+ copy("older", { savedAt: NOW - 2000 }),+ copy("old", { savedAt: NOW - 1000 }),+ ],+ failing: new Set(["new"]),+ });+ await synchronizeOfflineCopies([candidate("new")], next.deps);+ expect(next.ids()).toEqual(["old"]);+ });++ it("does not download articles that are still being crawled", async () => {+ const h = harness();+ const result = await synchronizeOfflineCopies(+ [candidate("pending", false)],+ h.deps,+ );+ expect(h.downloads).toEqual([]);+ expect(result.failed).toBe(1);+ });++ it("makes room by evicting lower-priority automatic copies only", async () => {+ const h = harness({+ budgetBytes: 30,+ stored: [copy("mine", { source: "manual" }), copy("b"), copy("c")],+ });+ await synchronizeOfflineCopies(+ [candidate("a"), candidate("b"), candidate("c")],+ h.deps,+ );+ expect(h.ids()).toEqual(["a", "b", "mine"]);+ });++ it("stops at the budget and keeps higher-priority copies", async () => {+ const h = harness({+ budgetBytes: 30,+ sizes: { c: 20 },+ stored: [copy("a"), copy("b")],+ });+ const result = await synchronizeOfflineCopies(+ [candidate("a"), candidate("b"), candidate("c"), candidate("d")],+ h.deps,+ );+ expect(h.ids()).toEqual(["a", "b"]);+ expect(h.downloads).toEqual(["c"]);+ expect(result).toMatchObject({ budgetReached: true, ready: 2, total: 4 });++ // Next pass: the size is known, so it isn't downloaded again.+ await synchronizeOfflineCopies(+ [candidate("a"), candidate("b"), candidate("c")],+ h.deps,+ );+ expect(h.downloads).toEqual(["c"]);+ });++ it("shrinks to a lowered budget without touching manual copies", async () => {+ const h = harness({+ budgetBytes: 15,+ stored: [+ copy("mine", { source: "manual", byteSize: 20 }),+ copy("a"),+ copy("b"),+ ],+ });+ await synchronizeOfflineCopies([candidate("a"), candidate("b")], h.deps);+ expect(h.ids()).toEqual(["mine"]);+ // Their sizes were known, so they weren't downloaded again to check.+ expect(h.downloads).toEqual([]);+ });++ it("does not evict anything when saving fails", async () => {+ const h = harness({ budgetBytes: 20, stored: [copy("b"), copy("c")] });+ const prepare = h.deps.prepare;+ h.deps.prepare = async (next) => ({+ ...(await prepare(next)),+ save: () => {+ throw new Error("disk full");+ },+ });+ const result = await synchronizeOfflineCopies(+ [candidate("a"), candidate("b"), candidate("c")],+ h.deps,+ );+ expect(h.ids()).toEqual(["b", "c"]);+ expect(result.failed).toBe(1);+ });++ it("stops without cleaning up when interrupted", async () => {+ let calls = 0;+ const h = harness({+ stored: [copy("old")],+ checkpoint: async () => {+ calls += 1;+ // The first two are before anything changes and before "a".+ if (calls > 2) {+ throw new SyncInterrupted("offline");+ }+ },+ });+ await expect(+ synchronizeOfflineCopies([candidate("a"), candidate("b")], h.deps),+ ).rejects.toBeInstanceOf(SyncInterrupted);+ expect(h.ids()).toEqual(["a", "old"]);+ });++ it("touches nothing when the pass must stop before it starts", async () => {+ const h = harness({+ budgetBytes: 5,+ stored: [copy("a"), copy("gone")],+ checkpoint: async () => {+ throw new SyncInterrupted("cancelled");+ },+ });+ await expect(+ synchronizeOfflineCopies([candidate("a")], h.deps),+ ).rejects.toBeInstanceOf(SyncInterrupted);+ expect(h.ids()).toEqual(["a", "gone"]);+ });++ it("treats a failed download as an interruption when the pass must stop", async () => {+ let offline = false;+ const h = harness({+ failing: new Set(["a"]),+ checkpoint: async () => {+ if (offline) {+ throw new SyncInterrupted("offline");+ }+ },+ });+ const prepare = h.deps.prepare;+ h.deps.prepare = async (next) => {+ offline = true;+ return prepare(next);+ };+ await expect(+ synchronizeOfflineCopies([candidate("a")], h.deps),+ ).rejects.toBeInstanceOf(SyncInterrupted);+ });++ it("skips articles excluded while the pass runs", async () => {+ const h = harness();+ const prepare = h.deps.prepare;+ h.deps.prepare = async (next) => {+ h.excluded.add(next.bookmarkId);+ return prepare(next);+ };+ await synchronizeOfflineCopies([candidate("a")], h.deps);+ expect(h.ids()).toEqual([]);+ });+});diff --git a/apps/mobile/lib/offlineSyncEngine.ts b/apps/mobile/lib/offlineSyncEngine.tsnew file mode 100644index 00000000..720749bd--- /dev/null+++ b/apps/mobile/lib/offlineSyncEngine.ts@@ -0,0 +1,315 @@+import type { ZBookmark } from "@karakeep/shared/types/bookmarks";+import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";+import { isBookmarkStillCrawling } from "@karakeep/shared/utils/bookmarkUtils";++import type { OfflineLibraryItem } from "./offlineLibrary";++export interface SyncCandidate {+ bookmarkId: string;+ // False while the server has not produced reader content yet.+ ready: boolean;+}++export type StoredCopy = Pick<+ OfflineLibraryItem,+ "bookmarkId" | "source" | "byteSize" | "savedAt"+>;++export interface PreparedCopy {+ byteSize: number;+ save: () => void;+}++export interface SyncProgress {+ total: number;+ ready: number;+ failed: number;+ budgetReached: boolean;+}++// Thrown when the pass must stop as a whole (cancelled, offline…), as opposed+// to a single article failing.+export class SyncInterrupted extends Error {+ constructor(readonly reason: "cancelled" | "offline" | "wifi") {+ super(`Offline downloads interrupted: ${reason}`);+ }+}++// Thrown by `prepare` when the server has no reader content for the bookmark+// (a crawl can succeed, or fail, without any).+export class ContentUnavailable extends Error {}++export interface SyncDeps {+ budgetBytes: number;+ listStored: () => StoredCopy[];+ prepare: (candidate: SyncCandidate) => Promise<PreparedCopy>;+ remove: (bookmarkId: string) => void;+ // Exclusions made while the pass runs; the candidates predate them.+ isExcluded: (bookmarkId: string) => boolean;+ // Throws SyncInterrupted when the pass has to stop.+ checkpoint: () => Promise<void>;+ // Sizes learnt on earlier passes, keyed by bookmark. Lets a pass skip an+ // article it already knows can't fit instead of downloading it again just+ // to find out.+ knownSizes: Map<string, number>;+ onProgress?: (progress: SyncProgress) => void;+}++function toSyncCandidate(bookmark: ZBookmark): SyncCandidate | undefined {+ const { type } = bookmark.content;+ if (type !== BookmarkTypes.LINK && type !== BookmarkTypes.TEXT) {+ return undefined;+ }+ // A failed crawl may have left an earlier crawl's content, so it's tried+ // like any other.+ return { bookmarkId: bookmark.id, ready: !isBookmarkStillCrawling(bookmark) };+}++/** Whether the bookmark is an article or note, not an attachment or a failed crawl. */+export function isDownloadable(bookmark: ZBookmark) {+ return (+ toSyncCandidate(bookmark) !== undefined &&+ !(+ bookmark.content.type === BookmarkTypes.LINK &&+ bookmark.content.crawlStatus === "failure"+ )+ );+}++function newestFirst(left: ZBookmark, right: ZBookmark) {+ return right.createdAt.getTime() - left.createdAt.getTime();+}++/**+ * Orders what should be kept offline, most important first: everything in the+ * kept lists, then the recent articles. Position is priority when the budget+ * runs out.+ */+export function selectCandidates(+ listBookmarks: ZBookmark[],+ recentBookmarks: ZBookmark[],+ excludedIds: readonly string[],+): SyncCandidate[] {+ const excluded = new Set(excludedIds);+ const seen = new Set<string>();+ const candidates: SyncCandidate[] = [];+ for (const bookmark of [+ ...[...listBookmarks].sort(newestFirst),+ ...[...recentBookmarks].sort(newestFirst),+ ]) {+ if (seen.has(bookmark.id) || excluded.has(bookmark.id)) {+ continue;+ }+ seen.add(bookmark.id);+ const candidate = toSyncCandidate(bookmark);+ if (candidate) {+ candidates.push(candidate);+ }+ }+ return candidates;+}++function totalSize(copies: StoredCopy[]) {+ return copies.reduce((sum, copy) => sum + (copy.byteSize ?? 0), 0);+}++/**+ * Automatic copies ranked below `rank`, least important first. Copies nobody+ * wants any more go first, oldest of them first.+ */+function evictionOrder(+ stored: StoredCopy[],+ priority: (bookmarkId: string) => number,+ rank: number,+) {+ return stored+ .filter(+ (copy) => copy.source === "auto" && priority(copy.bookmarkId) > rank,+ )+ .sort(+ (left, right) =>+ priority(right.bookmarkId) - priority(left.bookmarkId) ||+ left.savedAt - right.savedAt,+ );+}++/**+ * The copies to evict so that `incoming` fits, or undefined if it can't fit+ * even then. Never partial: nothing is worth evicting for an article that+ * will not be saved anyway.+ */+export function planRoomFor(+ incoming: { bookmarkId: string; byteSize: number; rank: number },+ stored: StoredCopy[],+ priority: (bookmarkId: string) => number,+ budgetBytes: number,+): string[] | undefined {+ let used = totalSize(stored) + incoming.byteSize;+ const evictions: string[] = [];+ for (const copy of evictionOrder(stored, priority, incoming.rank)) {+ if (used <= budgetBytes) {+ break;+ }+ evictions.push(copy.bookmarkId);+ used -= copy.byteSize ?? 0;+ }+ return used <= budgetBytes ? evictions : undefined;+}++/**+ * Brings the stored automatic copies in line with `candidates` within the+ * budget: downloads the ones that are missing and removes the ones no longer+ * wanted. Copies are downloaded once and not refreshed. Manual copies are+ * never touched. A download is saved before anything is evicted for it, so a+ * failure never loses what was there.+ */+export async function synchronizeOfflineCopies(+ candidates: SyncCandidate[],+ deps: SyncDeps,+): Promise<SyncProgress> {+ const ranks = new Map(+ candidates.map((candidate, index) => [candidate.bookmarkId, index]),+ );+ const priority = (bookmarkId: string) =>+ ranks.get(bookmarkId) ?? Number.POSITIVE_INFINITY;+ let failed = 0;+ let budgetReached = false;+ // Articles the server had no reader content for (see ContentUnavailable).+ // They don't count as failed, and are tried again on the next pass.+ const unavailable = new Set<string>();++ const progress = (): SyncProgress => {+ const stored = new Map(+ deps.listStored().map((copy) => [copy.bookmarkId, copy]),+ );+ let total = 0;+ let ready = 0;+ for (const { bookmarkId } of candidates) {+ if (stored.has(bookmarkId)) {+ ready += 1;+ } else if (unavailable.has(bookmarkId)) {+ // Nothing that could be downloaded, so nothing to wait for either.+ continue;+ }+ total += 1;+ }+ return { total, ready, failed, budgetReached };+ };+ const report = () => deps.onProgress?.(progress());+ const fail = () => {+ failed += 1;+ report();+ };++ // A pass that should already have stopped must not touch anything.+ await deps.checkpoint();++ // The budget may have been lowered since the last pass.+ let used = totalSize(deps.listStored());+ for (const copy of evictionOrder(deps.listStored(), priority, -1)) {+ if (used <= deps.budgetBytes) {+ break;+ }+ deps.remove(copy.bookmarkId);+ used -= copy.byteSize ?? 0;+ // Spares downloading it again below just to find it still doesn't fit.+ if (copy.byteSize !== undefined) {+ deps.knownSizes.set(copy.bookmarkId, copy.byteSize);+ }+ }+ report();++ for (const [rank, candidate] of candidates.entries()) {+ const { bookmarkId } = candidate;+ if (+ deps.isExcluded(bookmarkId) ||+ deps.listStored().some((copy) => copy.bookmarkId === bookmarkId)+ ) {+ continue;+ }+ if (!candidate.ready) {+ fail();+ continue;+ }++ const knownSize = deps.knownSizes.get(bookmarkId);+ if (+ knownSize !== undefined &&+ !planRoomFor(+ { bookmarkId, byteSize: knownSize, rank },+ deps.listStored(),+ priority,+ deps.budgetBytes,+ )+ ) {+ budgetReached = true;+ break;+ }++ // Only before network work: skipping stored copies needs no checks.+ await deps.checkpoint();+ let prepared: PreparedCopy;+ try {+ prepared = await deps.prepare(candidate);+ } catch (error) {+ if (error instanceof SyncInterrupted) {+ throw error;+ }+ if (error instanceof ContentUnavailable) {+ unavailable.add(bookmarkId);+ report();+ continue;+ }+ // A download that died because the connection went away is not the+ // article's fault; let the checkpoint stop the pass instead.+ await deps.checkpoint();+ fail();+ continue;+ }+ deps.knownSizes.set(bookmarkId, prepared.byteSize);++ // The user may have saved or removed it by hand while it downloaded.+ const stored = deps.listStored();+ if (+ deps.isExcluded(bookmarkId) ||+ stored.some((copy) => copy.bookmarkId === bookmarkId)+ ) {+ continue;+ }+ const evictions = planRoomFor(+ { bookmarkId, byteSize: prepared.byteSize, rank },+ stored,+ priority,+ deps.budgetBytes,+ );+ if (!evictions) {+ budgetReached = true;+ break;+ }+ try {+ prepared.save();+ } catch {+ fail();+ continue;+ }+ for (const bookmarkId of evictions) {+ deps.remove(bookmarkId);+ }+ report();+ }++ // Copies that fell out of the selection. While some articles failed, keep as+ // many of the old ones as are missing so a flaky pass doesn't shrink the+ // library, without letting one broken article block cleanup for good.+ const unwanted = deps+ .listStored()+ .filter((copy) => copy.source === "auto" && !ranks.has(copy.bookmarkId))+ .sort((left, right) => right.savedAt - left.savedAt);+ for (const copy of unwanted.slice(failed)) {+ deps.remove(copy.bookmarkId);+ }++ const result = progress();+ deps.onProgress?.(result);+ return result;+}