Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat(mobile): add the offline download sync engine (#3175)

Mohamed Bassema18fcb7c0f69
diff --git a/apps/mobile/lib/offlineSyncEngine.test.ts b/apps/mobile/lib/offlineSyncEngine.test.ts
new file mode 100644
index 00000000..07392463
--- /dev/null
+++ b/apps/mobile/lib/offlineSyncEngine.test.ts
@@ -0,0 +1,384 @@
+import { describe, expect, it } from "vitest";
+
+import type {
+ PreparedCopy,
+ StoredCopy,
+ SyncCandidate,
+ SyncDeps,
+} from "./offlineSyncEngine";
+import {
+ ContentUnavailable,
+ planRoomFor,
+ selectCandidates,
+ synchronizeOfflineCopies,
+ SyncInterrupted,
+} from "./offlineSyncEngine";
+import { assetBookmark, linkBookmark, textBookmark } from "./testing/bookmarks";
+
+const NOW = 1_000_000_000;
+
+function candidate(bookmarkId: string, ready = true): SyncCandidate {
+ return { bookmarkId, ready };
+}
+
+function copy(
+ bookmarkId: string,
+ overrides: Partial<StoredCopy> = {},
+): StoredCopy {
+ return {
+ bookmarkId,
+ source: "auto",
+ byteSize: 10,
+ savedAt: NOW - 1000,
+ ...overrides,
+ };
+}
+
+function harness({
+ stored = [],
+ budgetBytes = 1000,
+ sizes = {},
+ failing = new Set<string>(),
+ checkpoint = async () => undefined,
+}: {
+ stored?: StoredCopy[];
+ budgetBytes?: number;
+ sizes?: Record<string, number>;
+ failing?: Set<string>;
+ checkpoint?: () => Promise<void>;
+} = {}) {
+ const library = new Map(stored.map((item) => [item.bookmarkId, item]));
+ const downloads: string[] = [];
+ const excluded = new Set<string>();
+ const deps: SyncDeps = {
+ budgetBytes,
+ knownSizes: new Map(),
+ listStored: () => [...library.values()],
+ isExcluded: (bookmarkId) => excluded.has(bookmarkId),
+ checkpoint,
+ remove: (bookmarkId) => {
+ library.delete(bookmarkId);
+ },
+ prepare: async (next): Promise<PreparedCopy> => {
+ downloads.push(next.bookmarkId);
+ if (failing.has(next.bookmarkId)) {
+ throw new Error("download failed");
+ }
+ const byteSize = sizes[next.bookmarkId] ?? 10;
+ return {
+ byteSize,
+ save: () => {
+ library.set(next.bookmarkId, {
+ bookmarkId: next.bookmarkId,
+ source: "auto",
+ byteSize,
+ savedAt: NOW,
+ });
+ },
+ };
+ },
+ };
+ return {
+ deps,
+ downloads,
+ excluded,
+ ids: () => [...library.keys()].sort(),
+ };
+}
+
+describe("selectCandidates", () => {
+ it("puts list articles first, newest first, without duplicates", () => {
+ const result = selectCandidates(
+ [
+ linkBookmark("list-old", { createdAt: 1 }),
+ linkBookmark("both", { createdAt: 5 }),
+ ],
+ [
+ linkBookmark("recent", { createdAt: 9 }),
+ linkBookmark("both", { createdAt: 5 }),
+ ],
+ [],
+ );
+ expect(result.map((item) => item.bookmarkId)).toEqual([
+ "both",
+ "list-old",
+ "recent",
+ ]);
+ });
+
+ it("drops exclusions and assets, and marks pending crawls", () => {
+ // Older servers report neither a crawl status nor a crawl time.
+ const link = linkBookmark("uncrawled", { createdAt: 3 });
+ const uncrawled = {
+ ...link,
+ content: {
+ ...link.content,
+ crawlStatus: undefined,
+ crawledAt: undefined,
+ },
+ };
+ const result = selectCandidates(
+ [],
+ [
+ linkBookmark("excluded", { createdAt: 6 }),
+ assetBookmark("pdf", 5),
+ linkBookmark("broken", { createdAt: 4, crawlStatus: "failure" }),
+ linkBookmark("pending", { createdAt: 3, crawlStatus: "pending" }),
+ uncrawled,
+ textBookmark("note", 2),
+ ],
+ ["excluded"],
+ );
+ expect(result).toEqual([
+ // A failed crawl may have kept an earlier crawl's content.
+ { bookmarkId: "broken", ready: true },
+ { bookmarkId: "pending", ready: false },
+ { bookmarkId: "uncrawled", ready: false },
+ { bookmarkId: "note", ready: true },
+ ]);
+ });
+});
+
+describe("planRoomFor", () => {
+ const ranks = new Map([
+ ["a", 0],
+ ["b", 1],
+ ["c", 2],
+ ]);
+ const priority = (id: string) => ranks.get(id) ?? Number.POSITIVE_INFINITY;
+
+ it("evicts unwanted copies first, then the lowest priority", () => {
+ const stored = [
+ copy("b"),
+ copy("c"),
+ copy("gone"),
+ copy("mine", { source: "manual" }),
+ ];
+ expect(
+ planRoomFor(
+ { bookmarkId: "a", byteSize: 10, rank: 0 },
+ stored,
+ priority,
+ 30,
+ ),
+ ).toEqual(["gone", "c"]);
+ });
+
+ it("evicts nothing when even evicting everything allowed is not enough", () => {
+ const stored = [copy("a"), copy("c"), copy("mine", { source: "manual" })];
+ expect(
+ planRoomFor(
+ { bookmarkId: "b", byteSize: 25, rank: 1 },
+ stored,
+ priority,
+ 30,
+ ),
+ ).toBeUndefined();
+ });
+});
+
+describe("synchronizeOfflineCopies", () => {
+ it("downloads only what is missing", async () => {
+ const h = harness({
+ stored: [copy("stored"), copy("mine", { source: "manual" })],
+ });
+ const result = await synchronizeOfflineCopies(
+ [candidate("stored"), candidate("mine"), candidate("new")],
+ h.deps,
+ );
+ expect(h.downloads).toEqual(["new"]);
+ expect(result).toMatchObject({ total: 3, ready: 3, failed: 0 });
+ });
+
+ it("skips an article without reader content until the next pass", async () => {
+ const h = harness();
+ const prepare = h.deps.prepare;
+ h.deps.prepare = async (next) => {
+ if (next.bookmarkId === "empty") {
+ h.downloads.push(next.bookmarkId);
+ throw new ContentUnavailable();
+ }
+ return prepare(next);
+ };
+ const result = await synchronizeOfflineCopies(
+ [candidate("empty"), candidate("a")],
+ h.deps,
+ );
+ expect(result).toMatchObject({ total: 1, ready: 1, failed: 0 });
+
+ await synchronizeOfflineCopies([candidate("empty")], h.deps);
+ expect(h.downloads).toEqual(["empty", "a", "empty"]);
+ });
+
+ it("rotates out articles that left the selection", async () => {
+ const h = harness({
+ stored: [copy("old"), copy("mine", { source: "manual" })],
+ });
+ await synchronizeOfflineCopies([candidate("new")], h.deps);
+ expect(h.ids()).toEqual(["mine", "new"]);
+ });
+
+ it("keeps one old article per failed download, newest first", async () => {
+ const h = harness({
+ stored: [
+ copy("older", { savedAt: NOW - 2000 }),
+ copy("old", { savedAt: NOW - 1000 }),
+ ],
+ failing: new Set(["new"]),
+ });
+ await synchronizeOfflineCopies(
+ [candidate("new"), candidate("pending", false)],
+ h.deps,
+ );
+ expect(h.ids()).toEqual(["old", "older"]);
+
+ const next = harness({
+ stored: [
+ copy("older", { savedAt: NOW - 2000 }),
+ copy("old", { savedAt: NOW - 1000 }),
+ ],
+ failing: new Set(["new"]),
+ });
+ await synchronizeOfflineCopies([candidate("new")], next.deps);
+ expect(next.ids()).toEqual(["old"]);
+ });
+
+ it("does not download articles that are still being crawled", async () => {
+ const h = harness();
+ const result = await synchronizeOfflineCopies(
+ [candidate("pending", false)],
+ h.deps,
+ );
+ expect(h.downloads).toEqual([]);
+ expect(result.failed).toBe(1);
+ });
+
+ it("makes room by evicting lower-priority automatic copies only", async () => {
+ const h = harness({
+ budgetBytes: 30,
+ stored: [copy("mine", { source: "manual" }), copy("b"), copy("c")],
+ });
+ await synchronizeOfflineCopies(
+ [candidate("a"), candidate("b"), candidate("c")],
+ h.deps,
+ );
+ expect(h.ids()).toEqual(["a", "b", "mine"]);
+ });
+
+ it("stops at the budget and keeps higher-priority copies", async () => {
+ const h = harness({
+ budgetBytes: 30,
+ sizes: { c: 20 },
+ stored: [copy("a"), copy("b")],
+ });
+ const result = await synchronizeOfflineCopies(
+ [candidate("a"), candidate("b"), candidate("c"), candidate("d")],
+ h.deps,
+ );
+ expect(h.ids()).toEqual(["a", "b"]);
+ expect(h.downloads).toEqual(["c"]);
+ expect(result).toMatchObject({ budgetReached: true, ready: 2, total: 4 });
+
+ // Next pass: the size is known, so it isn't downloaded again.
+ await synchronizeOfflineCopies(
+ [candidate("a"), candidate("b"), candidate("c")],
+ h.deps,
+ );
+ expect(h.downloads).toEqual(["c"]);
+ });
+
+ it("shrinks to a lowered budget without touching manual copies", async () => {
+ const h = harness({
+ budgetBytes: 15,
+ stored: [
+ copy("mine", { source: "manual", byteSize: 20 }),
+ copy("a"),
+ copy("b"),
+ ],
+ });
+ await synchronizeOfflineCopies([candidate("a"), candidate("b")], h.deps);
+ expect(h.ids()).toEqual(["mine"]);
+ // Their sizes were known, so they weren't downloaded again to check.
+ expect(h.downloads).toEqual([]);
+ });
+
+ it("does not evict anything when saving fails", async () => {
+ const h = harness({ budgetBytes: 20, stored: [copy("b"), copy("c")] });
+ const prepare = h.deps.prepare;
+ h.deps.prepare = async (next) => ({
+ ...(await prepare(next)),
+ save: () => {
+ throw new Error("disk full");
+ },
+ });
+ const result = await synchronizeOfflineCopies(
+ [candidate("a"), candidate("b"), candidate("c")],
+ h.deps,
+ );
+ expect(h.ids()).toEqual(["b", "c"]);
+ expect(result.failed).toBe(1);
+ });
+
+ it("stops without cleaning up when interrupted", async () => {
+ let calls = 0;
+ const h = harness({
+ stored: [copy("old")],
+ checkpoint: async () => {
+ calls += 1;
+ // The first two are before anything changes and before "a".
+ if (calls > 2) {
+ throw new SyncInterrupted("offline");
+ }
+ },
+ });
+ await expect(
+ synchronizeOfflineCopies([candidate("a"), candidate("b")], h.deps),
+ ).rejects.toBeInstanceOf(SyncInterrupted);
+ expect(h.ids()).toEqual(["a", "old"]);
+ });
+
+ it("touches nothing when the pass must stop before it starts", async () => {
+ const h = harness({
+ budgetBytes: 5,
+ stored: [copy("a"), copy("gone")],
+ checkpoint: async () => {
+ throw new SyncInterrupted("cancelled");
+ },
+ });
+ await expect(
+ synchronizeOfflineCopies([candidate("a")], h.deps),
+ ).rejects.toBeInstanceOf(SyncInterrupted);
+ expect(h.ids()).toEqual(["a", "gone"]);
+ });
+
+ it("treats a failed download as an interruption when the pass must stop", async () => {
+ let offline = false;
+ const h = harness({
+ failing: new Set(["a"]),
+ checkpoint: async () => {
+ if (offline) {
+ throw new SyncInterrupted("offline");
+ }
+ },
+ });
+ const prepare = h.deps.prepare;
+ h.deps.prepare = async (next) => {
+ offline = true;
+ return prepare(next);
+ };
+ await expect(
+ synchronizeOfflineCopies([candidate("a")], h.deps),
+ ).rejects.toBeInstanceOf(SyncInterrupted);
+ });
+
+ it("skips articles excluded while the pass runs", async () => {
+ const h = harness();
+ const prepare = h.deps.prepare;
+ h.deps.prepare = async (next) => {
+ h.excluded.add(next.bookmarkId);
+ return prepare(next);
+ };
+ await synchronizeOfflineCopies([candidate("a")], h.deps);
+ expect(h.ids()).toEqual([]);
+ });
+});
diff --git a/apps/mobile/lib/offlineSyncEngine.ts b/apps/mobile/lib/offlineSyncEngine.ts
new file mode 100644
index 00000000..720749bd
--- /dev/null
+++ b/apps/mobile/lib/offlineSyncEngine.ts
@@ -0,0 +1,315 @@
+import type { ZBookmark } from "@karakeep/shared/types/bookmarks";
+import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+import { isBookmarkStillCrawling } from "@karakeep/shared/utils/bookmarkUtils";
+
+import type { OfflineLibraryItem } from "./offlineLibrary";
+
+export interface SyncCandidate {
+ bookmarkId: string;
+ // False while the server has not produced reader content yet.
+ ready: boolean;
+}
+
+export type StoredCopy = Pick<
+ OfflineLibraryItem,
+ "bookmarkId" | "source" | "byteSize" | "savedAt"
+>;
+
+export interface PreparedCopy {
+ byteSize: number;
+ save: () => void;
+}
+
+export interface SyncProgress {
+ total: number;
+ ready: number;
+ failed: number;
+ budgetReached: boolean;
+}
+
+// Thrown when the pass must stop as a whole (cancelled, offline…), as opposed
+// to a single article failing.
+export class SyncInterrupted extends Error {
+ constructor(readonly reason: "cancelled" | "offline" | "wifi") {
+ super(`Offline downloads interrupted: ${reason}`);
+ }
+}
+
+// Thrown by `prepare` when the server has no reader content for the bookmark
+// (a crawl can succeed, or fail, without any).
+export class ContentUnavailable extends Error {}
+
+export interface SyncDeps {
+ budgetBytes: number;
+ listStored: () => StoredCopy[];
+ prepare: (candidate: SyncCandidate) => Promise<PreparedCopy>;
+ remove: (bookmarkId: string) => void;
+ // Exclusions made while the pass runs; the candidates predate them.
+ isExcluded: (bookmarkId: string) => boolean;
+ // Throws SyncInterrupted when the pass has to stop.
+ checkpoint: () => Promise<void>;
+ // Sizes learnt on earlier passes, keyed by bookmark. Lets a pass skip an
+ // article it already knows can't fit instead of downloading it again just
+ // to find out.
+ knownSizes: Map<string, number>;
+ onProgress?: (progress: SyncProgress) => void;
+}
+
+function toSyncCandidate(bookmark: ZBookmark): SyncCandidate | undefined {
+ const { type } = bookmark.content;
+ if (type !== BookmarkTypes.LINK && type !== BookmarkTypes.TEXT) {
+ return undefined;
+ }
+ // A failed crawl may have left an earlier crawl's content, so it's tried
+ // like any other.
+ return { bookmarkId: bookmark.id, ready: !isBookmarkStillCrawling(bookmark) };
+}
+
+/** Whether the bookmark is an article or note, not an attachment or a failed crawl. */
+export function isDownloadable(bookmark: ZBookmark) {
+ return (
+ toSyncCandidate(bookmark) !== undefined &&
+ !(
+ bookmark.content.type === BookmarkTypes.LINK &&
+ bookmark.content.crawlStatus === "failure"
+ )
+ );
+}
+
+function newestFirst(left: ZBookmark, right: ZBookmark) {
+ return right.createdAt.getTime() - left.createdAt.getTime();
+}
+
+/**
+ * Orders what should be kept offline, most important first: everything in the
+ * kept lists, then the recent articles. Position is priority when the budget
+ * runs out.
+ */
+export function selectCandidates(
+ listBookmarks: ZBookmark[],
+ recentBookmarks: ZBookmark[],
+ excludedIds: readonly string[],
+): SyncCandidate[] {
+ const excluded = new Set(excludedIds);
+ const seen = new Set<string>();
+ const candidates: SyncCandidate[] = [];
+ for (const bookmark of [
+ ...[...listBookmarks].sort(newestFirst),
+ ...[...recentBookmarks].sort(newestFirst),
+ ]) {
+ if (seen.has(bookmark.id) || excluded.has(bookmark.id)) {
+ continue;
+ }
+ seen.add(bookmark.id);
+ const candidate = toSyncCandidate(bookmark);
+ if (candidate) {
+ candidates.push(candidate);
+ }
+ }
+ return candidates;
+}
+
+function totalSize(copies: StoredCopy[]) {
+ return copies.reduce((sum, copy) => sum + (copy.byteSize ?? 0), 0);
+}
+
+/**
+ * Automatic copies ranked below `rank`, least important first. Copies nobody
+ * wants any more go first, oldest of them first.
+ */
+function evictionOrder(
+ stored: StoredCopy[],
+ priority: (bookmarkId: string) => number,
+ rank: number,
+) {
+ return stored
+ .filter(
+ (copy) => copy.source === "auto" && priority(copy.bookmarkId) > rank,
+ )
+ .sort(
+ (left, right) =>
+ priority(right.bookmarkId) - priority(left.bookmarkId) ||
+ left.savedAt - right.savedAt,
+ );
+}
+
+/**
+ * The copies to evict so that `incoming` fits, or undefined if it can't fit
+ * even then. Never partial: nothing is worth evicting for an article that
+ * will not be saved anyway.
+ */
+export function planRoomFor(
+ incoming: { bookmarkId: string; byteSize: number; rank: number },
+ stored: StoredCopy[],
+ priority: (bookmarkId: string) => number,
+ budgetBytes: number,
+): string[] | undefined {
+ let used = totalSize(stored) + incoming.byteSize;
+ const evictions: string[] = [];
+ for (const copy of evictionOrder(stored, priority, incoming.rank)) {
+ if (used <= budgetBytes) {
+ break;
+ }
+ evictions.push(copy.bookmarkId);
+ used -= copy.byteSize ?? 0;
+ }
+ return used <= budgetBytes ? evictions : undefined;
+}
+
+/**
+ * Brings the stored automatic copies in line with `candidates` within the
+ * budget: downloads the ones that are missing and removes the ones no longer
+ * wanted. Copies are downloaded once and not refreshed. Manual copies are
+ * never touched. A download is saved before anything is evicted for it, so a
+ * failure never loses what was there.
+ */
+export async function synchronizeOfflineCopies(
+ candidates: SyncCandidate[],
+ deps: SyncDeps,
+): Promise<SyncProgress> {
+ const ranks = new Map(
+ candidates.map((candidate, index) => [candidate.bookmarkId, index]),
+ );
+ const priority = (bookmarkId: string) =>
+ ranks.get(bookmarkId) ?? Number.POSITIVE_INFINITY;
+ let failed = 0;
+ let budgetReached = false;
+ // Articles the server had no reader content for (see ContentUnavailable).
+ // They don't count as failed, and are tried again on the next pass.
+ const unavailable = new Set<string>();
+
+ const progress = (): SyncProgress => {
+ const stored = new Map(
+ deps.listStored().map((copy) => [copy.bookmarkId, copy]),
+ );
+ let total = 0;
+ let ready = 0;
+ for (const { bookmarkId } of candidates) {
+ if (stored.has(bookmarkId)) {
+ ready += 1;
+ } else if (unavailable.has(bookmarkId)) {
+ // Nothing that could be downloaded, so nothing to wait for either.
+ continue;
+ }
+ total += 1;
+ }
+ return { total, ready, failed, budgetReached };
+ };
+ const report = () => deps.onProgress?.(progress());
+ const fail = () => {
+ failed += 1;
+ report();
+ };
+
+ // A pass that should already have stopped must not touch anything.
+ await deps.checkpoint();
+
+ // The budget may have been lowered since the last pass.
+ let used = totalSize(deps.listStored());
+ for (const copy of evictionOrder(deps.listStored(), priority, -1)) {
+ if (used <= deps.budgetBytes) {
+ break;
+ }
+ deps.remove(copy.bookmarkId);
+ used -= copy.byteSize ?? 0;
+ // Spares downloading it again below just to find it still doesn't fit.
+ if (copy.byteSize !== undefined) {
+ deps.knownSizes.set(copy.bookmarkId, copy.byteSize);
+ }
+ }
+ report();
+
+ for (const [rank, candidate] of candidates.entries()) {
+ const { bookmarkId } = candidate;
+ if (
+ deps.isExcluded(bookmarkId) ||
+ deps.listStored().some((copy) => copy.bookmarkId === bookmarkId)
+ ) {
+ continue;
+ }
+ if (!candidate.ready) {
+ fail();
+ continue;
+ }
+
+ const knownSize = deps.knownSizes.get(bookmarkId);
+ if (
+ knownSize !== undefined &&
+ !planRoomFor(
+ { bookmarkId, byteSize: knownSize, rank },
+ deps.listStored(),
+ priority,
+ deps.budgetBytes,
+ )
+ ) {
+ budgetReached = true;
+ break;
+ }
+
+ // Only before network work: skipping stored copies needs no checks.
+ await deps.checkpoint();
+ let prepared: PreparedCopy;
+ try {
+ prepared = await deps.prepare(candidate);
+ } catch (error) {
+ if (error instanceof SyncInterrupted) {
+ throw error;
+ }
+ if (error instanceof ContentUnavailable) {
+ unavailable.add(bookmarkId);
+ report();
+ continue;
+ }
+ // A download that died because the connection went away is not the
+ // article's fault; let the checkpoint stop the pass instead.
+ await deps.checkpoint();
+ fail();
+ continue;
+ }
+ deps.knownSizes.set(bookmarkId, prepared.byteSize);
+
+ // The user may have saved or removed it by hand while it downloaded.
+ const stored = deps.listStored();
+ if (
+ deps.isExcluded(bookmarkId) ||
+ stored.some((copy) => copy.bookmarkId === bookmarkId)
+ ) {
+ continue;
+ }
+ const evictions = planRoomFor(
+ { bookmarkId, byteSize: prepared.byteSize, rank },
+ stored,
+ priority,
+ deps.budgetBytes,
+ );
+ if (!evictions) {
+ budgetReached = true;
+ break;
+ }
+ try {
+ prepared.save();
+ } catch {
+ fail();
+ continue;
+ }
+ for (const bookmarkId of evictions) {
+ deps.remove(bookmarkId);
+ }
+ report();
+ }
+
+ // Copies that fell out of the selection. While some articles failed, keep as
+ // many of the old ones as are missing so a flaky pass doesn't shrink the
+ // library, without letting one broken article block cleanup for good.
+ const unwanted = deps
+ .listStored()
+ .filter((copy) => copy.source === "auto" && !ranks.has(copy.bookmarkId))
+ .sort((left, right) => right.savedAt - left.savedAt);
+ for (const copy of unwanted.slice(failed)) {
+ deps.remove(copy.bookmarkId);
+ }
+
+ const result = progress();
+ deps.onProgress?.(result);
+ return result;
+}