⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
feat(mobile): implement full offline storage and sync functionality for the preserved formats
daniel31x13f6c512cce2d0
diff --git a/apps/mobile/app/(tabs)/settings/index.tsx b/apps/mobile/app/(tabs)/settings/index.tsxindex 6c1cd7f6..b6a2cb15 100644--- a/apps/mobile/app/(tabs)/settings/index.tsx+++ b/apps/mobile/app/(tabs)/settings/index.tsx@@ -8,6 +8,8 @@ import {TouchableOpacity,Platform,Alert,+ Switch,+ ActivityIndicator,} from "react-native";import { nativeApplicationVersion } from "expo-application";import { useColorScheme } from "nativewind";@@ -17,14 +19,24 @@ import {AppWindowMac,Check,ChevronRight,+ Download,ExternalLink,Folder,+ HardDrive,LogOut,Mail,Moon,+ RefreshCw,Smartphone,Sun,+ Trash2,} from "lucide-react-native";+import { clearCache } from "@/lib/cache";+import {+ formatBytes,+ recomputeStorage,+ useOfflineSyncStore,+} from "@/lib/offlineSync";import useDataStore from "@/store/data";import * as Clipboard from "expo-clipboard";import { useRouter } from "expo-router";@@ -45,6 +57,21 @@ export default function SettingsScreen() {const router = useRouter();+ const syncStatus = useOfflineSyncStore((s) => s.status);+ const syncProcessed = useOfflineSyncStore((s) => s.processed);+ const syncTotal = useOfflineSyncStore((s) => s.total);+ const bytesUsed = useOfflineSyncStore((s) => s.bytesUsed);+ const syncPercent =+ syncTotal > 0 ? Math.floor((syncProcessed / syncTotal) * 100) : null;+ const syncStatusLabel =+ syncStatus === "paused"+ ? "Waiting for connection"+ : syncPercent === null+ ? "Preparing…"+ : syncPercent >= 100+ ? "Up to date"+ : `${syncPercent}%`;+return (<Viewstyle={styles.container}@@ -228,6 +255,111 @@ export default function SettingsScreen() {</View></View>+ <View>+ <Text className="mb-4 mx-4 text-neutral">Offline Storage</Text>+ <View className="bg-base-200 rounded-xl flex-col">+ <View className="py-3 px-4">+ <View className="flex-row gap-2 items-center justify-between">+ <View className="flex-row items-center gap-2">+ <Download+ size={20}+ color={rawTheme[colorScheme as ThemeName].neutral}+ />+ <Text className="text-base-content">+ Save for offline access+ </Text>+ </View>+ <Switch+ value={!!data.offlineEnabled}+ onValueChange={(value) =>+ updateData({ offlineEnabled: value })+ }+ trackColor={{+ true: rawTheme[colorScheme as ThemeName].primary,+ }}+ thumbColor={rawTheme[colorScheme as ThemeName]["base-100"]}+ />+ </View>+ <Text className="text-sm text-neutral mt-2 ml-7">+ Automatically saves preserved formats for links loaded in the+ app to this device for offline access. When this is off, only+ formats you open are saved.+ </Text>+ </View>+ {data.offlineEnabled ? (+ <>+ <View className="h-px bg-neutral-content ml-12" />+ <View className="flex-row gap-2 items-center justify-between py-3 px-4">+ <View className="flex-row items-center gap-2">+ <RefreshCw+ size={20}+ color={rawTheme[colorScheme as ThemeName].neutral}+ />+ <Text className="text-base-content">Offline sync</Text>+ </View>+ <View className="flex-row items-center gap-2">+ {syncStatus === "syncing" ? (+ <ActivityIndicator+ size="small"+ color={rawTheme[colorScheme as ThemeName].primary}+ />+ ) : null}+ <Text className="text-neutral">{syncStatusLabel}</Text>+ </View>+ </View>+ </>+ ) : null}+ <View className="h-px bg-neutral-content ml-12" />+ <View className="flex-row gap-2 items-center justify-between py-3 px-4">+ <View className="flex-row items-center gap-2">+ <HardDrive+ size={20}+ color={rawTheme[colorScheme as ThemeName].neutral}+ />+ <Text className="text-base-content">Storage used</Text>+ </View>+ <Text className="text-neutral">{formatBytes(bytesUsed)}</Text>+ </View>+ <View className="h-px bg-neutral-content ml-12" />+ <TouchableOpacity+ className="flex-row gap-2 items-center justify-between py-3 px-4"+ disabled={syncStatus === "syncing"}+ onPress={() =>+ Alert.alert(+ "Clear cache",+ "This will delete all downloaded formats and previews from this device.",+ [+ { text: "Cancel", style: "cancel" },+ {+ text: "Clear",+ style: "destructive",+ onPress: async () => {+ await clearCache();+ useOfflineSyncStore.getState().reset();+ await recomputeStorage();+ },+ },+ ]+ )+ }+ >+ <View className="flex-row items-center gap-2">+ <Trash2+ size={20}+ color={syncStatus === "syncing" ? "gray" : "red"}+ />+ <Text+ className={+ syncStatus === "syncing" ? "text-neutral" : "text-red-500"+ }+ >+ Clear cache+ </Text>+ </View>+ </TouchableOpacity>+ </View>+ </View>+<View><Text className="mb-4 mx-4 text-neutral">Contact Us</Text><View className="bg-base-200 rounded-xl flex-col">diff --git a/apps/mobile/app/(tabs)/tags/index.tsx b/apps/mobile/app/(tabs)/tags/index.tsxindex 151e0a93..bfe1e745 100644--- a/apps/mobile/app/(tabs)/tags/index.tsx+++ b/apps/mobile/app/(tabs)/tags/index.tsx@@ -16,6 +16,8 @@ import { useColorScheme } from "nativewind";import { TagSort } from "@linkwarden/types/global";import { useTags } from "@linkwarden/router/tags";import { isAtLeastInstanceVersion, useConfig } from "@linkwarden/router/config";+import { useQueryClient } from "@tanstack/react-query";+import { resetInfiniteQueryPagination } from "@linkwarden/router/lib";const MIN_TAG_SEARCH_VERSION = "2.14.1";@@ -34,6 +36,7 @@ export default function TagsScreen() {const { auth } = useAuthStore();const { search } = useLocalSearchParams<{ search?: string }>();const config = useConfig(auth);+ const queryClient = useQueryClient();const searchQuery = decodeSearchParam(search);const supportsTagSearch = isAtLeastInstanceVersion(config.data?.INSTANCE_VERSION,@@ -72,7 +75,9 @@ export default function TagsScreen() {refreshControl={<Spinnerrefreshing={tags.isRefetching}- onRefresh={() => tags.refetch()}+ onRefresh={() =>+ resetInfiniteQueryPagination(queryClient, ["tags"])+ }progressBackgroundColor={rawTheme[colorScheme as ThemeName]["base-200"]}diff --git a/apps/mobile/app/_layout.tsx b/apps/mobile/app/_layout.tsxindex 3266a827..36df19bc 100644--- a/apps/mobile/app/_layout.tsx+++ b/apps/mobile/app/_layout.tsx@@ -22,6 +22,7 @@ import { KeyboardProvider } from "react-native-keyboard-controller";import { queryClient } from "@/lib/queryClient";import { StatusBar } from "expo-status-bar";import * as Sentry from "@sentry/react-native";+import OfflineSyncProvider from "@/components/OfflineSyncProvider";Sentry.init({dsn: "https://00d7eed9e810cbbf91a7ed3547e37100@o4510998442475520.ingest.us.sentry.io/4511033679609856",@@ -90,7 +91,7 @@ export default Sentry.wrap(function RootLayout() {maxAge: Infinity,dehydrateOptions: {shouldDehydrateMutation: () => true,- shouldDehydrateQuery: () => true,+ shouldDehydrateQuery: (query) => query.state.status === "success",},}}onSuccess={() => {@@ -112,6 +113,7 @@ const RootComponent = ({ isLoading }: { isLoading: boolean }) => {style={[{ flex: 1 }, colorScheme === "dark" ? darkTheme : lightTheme]}><SheetProvider>+ <OfflineSyncProvider /><StatusBarstyle={colorScheme === "dark" ? "light" : "dark"}backgroundColor={rawTheme[colorScheme as ThemeName]["base-100"]}diff --git a/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx b/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsxindex 43e9e79f..22fde5c8 100644--- a/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx+++ b/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx@@ -1,7 +1,6 @@import { useEffect, useState, type ReactNode } from "react";import {ActivityIndicator,- Image,Linking,Text,TouchableOpacity,@@ -14,14 +13,12 @@ import ActionSheet, {SheetProps,} from "react-native-actions-sheet";import { decode } from "html-entities";-import * as Clipboard from "expo-clipboard";import { useRouter } from "expo-router";import { useColorScheme } from "nativewind";import { useSafeAreaInsets } from "react-native-safe-area-context";import {CalendarDays,ChevronRight,- Copy,ExternalLink,File,FileCode,@@ -360,7 +357,10 @@ export default function LinkDetailsSheet(<View className="rounded-lg bg-base-100 px-4 py-3">{link.description ? (- <Text className="text-base leading-6 text-base-content">+ <Text+ className="text-base leading-6 text-base-content"+ selectable+ >{decode(link.description)}</Text>) : (diff --git a/apps/mobile/components/Links.tsx b/apps/mobile/components/Links.tsxindex b08eb219..95ae3b16 100644--- a/apps/mobile/components/Links.tsx+++ b/apps/mobile/components/Links.tsx@@ -11,6 +11,8 @@ import { LinkIncludingShortenedCollectionAndTags } from "@linkwarden/types/globaimport Spinner from "@/components/ui/Spinner";import { rawTheme, ThemeName } from "@/lib/colors";import { useColorScheme } from "nativewind";+import { useQueryClient } from "@tanstack/react-query";+import { resetInfiniteQueryPagination } from "@linkwarden/router/lib";const RenderItem = React.memo(({ item }: { item: LinkIncludingShortenedCollectionAndTags }) => {@@ -25,6 +27,7 @@ type Props = {export default function Links({ links, data }: Props) {const { colorScheme } = useColorScheme();+ const queryClient = useQueryClient();const [promptedRefetch, setPromptedRefetch] = useState(false);return data.isLoading ? (@@ -42,7 +45,7 @@ export default function Links({ links, data }: Props) {refreshing={data.isRefetching && promptedRefetch}onRefresh={async () => {setPromptedRefetch(true);- await data.refetch();+ await resetInfiniteQueryPagination(queryClient, ["links"]);setPromptedRefetch(false);}}progressBackgroundColor={diff --git a/apps/mobile/components/OfflineSyncProvider.tsx b/apps/mobile/components/OfflineSyncProvider.tsxnew file mode 100644index 00000000..8ceb6b42--- /dev/null+++ b/apps/mobile/components/OfflineSyncProvider.tsx@@ -0,0 +1,56 @@+import { useEffect } from "react";+import { useConfig } from "@linkwarden/router/config";+import useAuthStore from "@/store/auth";+import useDataStore from "@/store/data";+import {+ recomputeStorage,+ startSync,+ stopSync,+ subscribeToConnectivity,+ subscribeToQueryCache,+ unsubscribeFromConnectivity,+ unsubscribeFromQueryCache,+} from "@/lib/offlineSync";++export default function OfflineSyncProvider() {+ const { auth } = useAuthStore();+ const { data } = useDataStore();+ const config = useConfig(auth);++ useEffect(() => {+ subscribeToConnectivity();+ recomputeStorage();++ return () => unsubscribeFromConnectivity();+ }, []);++ useEffect(() => {+ if (auth.status !== "authenticated") {+ stopSync();+ unsubscribeFromQueryCache();+ return;+ }++ if (!data.offlineEnabled) {+ stopSync();+ unsubscribeFromQueryCache();+ return;+ }++ if (config.isLoading) return;++ startSync(auth, config.data?.USER_CONTENT_DOMAIN);+ subscribeToQueryCache();++ return () => stopSync();+ }, [+ auth.status,+ auth.instance,+ auth.session,+ data.offlineEnabled,+ config.isLoading,+ config.data?.USER_CONTENT_DOMAIN,+ ]);++ return null;+}diff --git a/apps/mobile/lib/cache.ts b/apps/mobile/lib/cache.tsindex 6b9b01d7..3f674e61 100644--- a/apps/mobile/lib/cache.ts+++ b/apps/mobile/lib/cache.ts@@ -1,5 +1,7 @@import * as FileSystem from "expo-file-system/legacy";import NetInfo from "@react-native-community/netinfo";+import { ArchivedFormat, MobileAuth } from "@linkwarden/types/global";+import getPreservedFormatUrl from "@linkwarden/lib/getPreservedFormatUrl";type LoadCacheOrFetchOptions<T> = {filePath: string;@@ -55,6 +57,153 @@ export const loadCacheOrFetch = async <T = string>({}};+export type CacheFormat =+ | "readable"+ | "webpage"+ | "jpeg"+ | "png"+ | "pdf"+ | "preview";++export const getCachePathForFormat = (+ linkId: number,+ format: CacheFormat+): string => {+ switch (format) {+ case "readable":+ return (+ FileSystem.documentDirectory ++ `archivedData/readable/link_${linkId}.html`+ );+ case "webpage":+ return (+ FileSystem.documentDirectory ++ `archivedData/webpage/link_${linkId}.html`+ );+ case "jpeg":+ return (+ FileSystem.documentDirectory + `archivedData/jpeg/link_${linkId}.jpeg`+ );+ case "png":+ return (+ FileSystem.documentDirectory + `archivedData/png/link_${linkId}.png`+ );+ case "pdf":+ return (+ FileSystem.documentDirectory + `archivedData/pdf/link_${linkId}.pdf`+ );+ case "preview":+ return (+ FileSystem.documentDirectory ++ `archivedData/previews/link_${linkId}.jpg`+ );+ }+};++type FetchFormatToCacheOptions = {+ link: { id: number; updatedAt?: string | Date | null };+ format: CacheFormat;+ auth: MobileAuth;+ userContentDomain?: string | null;+};++export const fetchFormatToCache = async ({+ link,+ format,+ auth,+ userContentDomain,+}: FetchFormatToCacheOptions): Promise<{+ uri: string;+ size: number;+ delta: number;+}> => {+ const filePath = getCachePathForFormat(link.id, format);++ await FileSystem.makeDirectoryAsync(+ filePath.substring(0, filePath.lastIndexOf("/")),+ { intermediates: true }+ ).catch(() => {});++ const prev = await FileSystem.getInfoAsync(filePath);+ const prevSize = prev.exists ? (prev as any).size ?? 0 : 0;++ const archivedFormat =+ format === "readable"+ ? ArchivedFormat.readability+ : format === "webpage"+ ? ArchivedFormat.monolith+ : format === "jpeg"+ ? ArchivedFormat.jpeg+ : format === "png"+ ? ArchivedFormat.png+ : format === "pdf"+ ? ArchivedFormat.pdf+ : ArchivedFormat.jpeg;++ const previewSuffix =+ format === "preview" ? `&preview=true&updatedAt=${link.updatedAt}` : "";++ let apiUrl: string;+ let headers: Record<string, string> | undefined = {+ Authorization: `Bearer ${auth.session}`,+ };++ if (format === "webpage" && userContentDomain) {+ apiUrl = await getPreservedFormatUrl({+ tokenEndpoint: `${auth.instance}/api/v1/preserved/token`,+ linkId: link.id,+ format: archivedFormat,+ headers,+ });+ headers = undefined;+ } else {+ apiUrl = `${auth.instance}/api/v1/archives/${link.id}?format=${archivedFormat}${previewSuffix}`;+ }++ const tmpPath = `${filePath}.part`;+ await FileSystem.deleteAsync(tmpPath, { idempotent: true }).catch(() => {});++ try {+ if (format === "readable") {+ const response = await fetch(apiUrl, { headers });+ if (!response.ok) throw new Error(`HTTP ${response.status}`);+ const data = (await response.json()).content;++ await FileSystem.writeAsStringAsync(tmpPath, data, {+ encoding: FileSystem.EncodingType.UTF8,+ });+ } else {+ const result = await FileSystem.downloadAsync(apiUrl, tmpPath, {+ headers,+ });+ if (result.status < 200 || result.status >= 300) {+ throw new Error(`HTTP ${result.status}`);+ }+ }++ await FileSystem.deleteAsync(filePath, { idempotent: true }).catch(+ () => {}+ );+ await FileSystem.moveAsync({ from: tmpPath, to: filePath });++ const info = await FileSystem.getInfoAsync(filePath);+ const size = (info as any).size ?? 0;+ return { uri: filePath, size, delta: size - prevSize };+ } catch (e) {+ await FileSystem.deleteAsync(tmpPath, { idempotent: true }).catch(() => {});+ throw e;+ }+};++export const getCacheSize = async (): Promise<number> => {+ const root = FileSystem.documentDirectory + "archivedData";++ const rootInfo = await FileSystem.getInfoAsync(root);+ if (!rootInfo.exists) return 0;++ return rootInfo.size ?? 0;+};+export const clearCache = async () => {await Promise.all([FileSystem.deleteAsync(FileSystem.documentDirectory + "archivedData", {@@ -67,25 +216,13 @@ export const clearCache = async () => {};export const deleteLinkCache = async (linkId: number) => {- const readablePath =- FileSystem.documentDirectory + `archivedData/readable/link_${linkId}.html`;- const webpagePath =- FileSystem.documentDirectory + `archivedData/webpage/link_${linkId}.html`;- const jpegPath =- FileSystem.documentDirectory + `archivedData/jpeg/link_${linkId}.jpeg`;- const pngPath =- FileSystem.documentDirectory + `archivedData/png/link_${linkId}.png`;- const pdfPath =- FileSystem.documentDirectory + `archivedData/pdf/link_${linkId}.pdf`;- const previewPath =- FileSystem.documentDirectory + `archivedData/previews/link_${linkId}.jpg`;-- await Promise.all([- FileSystem.deleteAsync(readablePath, { idempotent: true }),- FileSystem.deleteAsync(webpagePath, { idempotent: true }),- FileSystem.deleteAsync(jpegPath, { idempotent: true }),- FileSystem.deleteAsync(pngPath, { idempotent: true }),- FileSystem.deleteAsync(pdfPath, { idempotent: true }),- FileSystem.deleteAsync(previewPath, { idempotent: true }),- ]);+ await Promise.all(+ (+ ["readable", "webpage", "jpeg", "png", "pdf", "preview"] as CacheFormat[]+ ).map((format) =>+ FileSystem.deleteAsync(getCachePathForFormat(linkId, format), {+ idempotent: true,+ })+ )+ );};diff --git a/apps/mobile/lib/offlineSync.ts b/apps/mobile/lib/offlineSync.tsnew file mode 100644index 00000000..91f485ed--- /dev/null+++ b/apps/mobile/lib/offlineSync.ts@@ -0,0 +1,578 @@+import { create } from "zustand";+import NetInfo from "@react-native-community/netinfo";+import {+ LinkIncludingShortenedCollectionAndTags,+ MobileAuth,+} from "@linkwarden/types/global";+import { formatAvailable } from "@linkwarden/lib/formatStats";+import * as FileSystem from "expo-file-system/legacy";+import { CacheFormat, fetchFormatToCache, getCacheSize } from "@/lib/cache";+import { queryClient } from "@/lib/queryClient";++type SyncStatus = "idle" | "syncing" | "paused";++type OfflineSyncState = {+ status: SyncStatus;+ processed: number;+ total: number;+ failed: number;+ currentLinkId: number | null;+ bytesUsed: number;+ setStatus: (status: SyncStatus) => void;+ setProgress: (processed: number, total: number) => void;+ incrementProcessed: () => void;+ incrementFailed: () => void;+ setCurrentLinkId: (id: number | null) => void;+ setBytesUsed: (bytes: number) => void;+ addBytes: (delta: number) => void;+ reset: () => void;+};++export const useOfflineSyncStore = create<OfflineSyncState>((set) => ({+ status: "idle",+ processed: 0,+ total: 0,+ failed: 0,+ currentLinkId: null,+ bytesUsed: 0,+ setStatus: (status) => set({ status }),+ setProgress: (processed, total) =>+ set((s) => ({ processed, total, failed: processed === 0 ? 0 : s.failed })),+ incrementProcessed: () => set((s) => ({ processed: s.processed + 1 })),+ incrementFailed: () => set((s) => ({ failed: s.failed + 1 })),+ setCurrentLinkId: (id) => set({ currentLinkId: id }),+ setBytesUsed: (bytes) => set({ bytesUsed: bytes }),+ addBytes: (delta) =>+ set((s) => ({ bytesUsed: Math.max(0, s.bytesUsed + delta) })),+ reset: () =>+ set({+ status: "idle",+ processed: 0,+ total: 0,+ failed: 0,+ currentLinkId: null,+ }),+}));++let cancelled = false;+let runPromise: Promise<void> | null = null;+let syncGeneration = 0;+let netUnsubscribe: (() => void) | null = null;+let cacheUnsubscribe: (() => void) | null = null;+let cacheSyncTimeout: ReturnType<typeof setTimeout> | null = null;+let activeAuth: MobileAuth | null = null;+let activeUserContentDomain: string | null | undefined = undefined;+let syncQueryWriteDepth = 0;+let ignoredQueryKeyClearTimeout: ReturnType<typeof setTimeout> | null = null;++const archiveRoot = FileSystem.documentDirectory + "archivedData";+const staleFileGraceMs = 5000;++const formatDirectories: Record<+ CacheFormat,+ { dirName: string; extension: string }+> = {+ readable: { dirName: "readable", extension: "html" },+ webpage: { dirName: "webpage", extension: "html" },+ jpeg: { dirName: "jpeg", extension: "jpeg" },+ png: { dirName: "png", extension: "png" },+ pdf: { dirName: "pdf", extension: "pdf" },+ preview: { dirName: "previews", extension: "jpg" },+};++type CachedFile = {+ path: string;+ size: number;+ linkId?: number;+ format?: CacheFormat;+ modifiedAt?: number;+ partial: boolean;+};++type CacheSnapshot = {+ bytesUsed: number;+ filesByLinkId: Map<number, CachedFile[]>;+ formatsByLinkId: Map<number, Map<CacheFormat, CachedFile>>;+ partialFiles: CachedFile[];+};++type LinksCacheSnapshot = {+ links: LinkIncludingShortenedCollectionAndTags[];+ hasLinkSource: boolean;+};++const getLinkFreshnessTime = (link: LinkIncludingShortenedCollectionAndTags) =>+ new Date((link as any)?.updatedAt).getTime();++const getFileModifiedAt = (info: any): number | undefined => {+ if (+ typeof info.modificationTime !== "number" ||+ !Number.isFinite(info.modificationTime)+ ) {+ return undefined;+ }++ return info.modificationTime > 10_000_000_000+ ? info.modificationTime+ : info.modificationTime * 1000;+};++const isOnline = (state: {+ isConnected: boolean | null;+ isInternetReachable?: boolean | null;+}) => state.isConnected === true && state.isInternetReachable !== false;++const ignoredQueryKeySignatures = new Set<string>();++const getQueryKeySignature = (queryKey: unknown) => {+ try {+ return JSON.stringify(queryKey);+ } catch {+ return undefined;+ }+};++const ignoreQueryKeyTemporarily = (queryKey: unknown) => {+ const signature = getQueryKeySignature(queryKey);+ if (!signature) return;++ ignoredQueryKeySignatures.add(signature);+ if (ignoredQueryKeyClearTimeout) clearTimeout(ignoredQueryKeyClearTimeout);+ ignoredQueryKeyClearTimeout = setTimeout(() => {+ ignoredQueryKeySignatures.clear();+ ignoredQueryKeyClearTimeout = null;+ }, 500);+};++const sameArchiveState = (a: any, b: LinkIncludingShortenedCollectionAndTags) =>+ a?.id === b.id &&+ a?.updatedAt === b.updatedAt &&+ a?.preview === b.preview &&+ a?.readable === (b as any).readable &&+ a?.monolith === (b as any).monolith &&+ a?.pdf === (b as any).pdf &&+ a?.image === (b as any).image;++const setCachedLinkQueryData = (+ link: LinkIncludingShortenedCollectionAndTags & { id: number }+) => {+ const queryKey = ["link", link.id, false] as const;+ const existing = queryClient.getQueryData(queryKey);+ if (sameArchiveState(existing, link)) return;++ ignoreQueryKeyTemporarily(queryKey);+ syncQueryWriteDepth += 1;+ try {+ queryClient.setQueryData(queryKey, link);+ } finally {+ syncQueryWriteDepth -= 1;+ }+};++const getLinksFromCache = (): LinksCacheSnapshot => {+ const linksById = new Map<number, LinkIncludingShortenedCollectionAndTags>();+ const add = (link: any) => {+ const id = Number(link?.id);+ if (!Number.isFinite(id)) return;++ const existing = linksById.get(id);+ if (+ !existing ||+ getLinkFreshnessTime(link) >= getLinkFreshnessTime(existing)+ ) {+ linksById.set(id, link);+ }+ };+ let hasLinkSource = false;++ for (const query of queryClient.getQueryCache().getAll()) {+ const key = query.queryKey;+ const data = query.state.data as any;+ if (!data) continue;+ if (!Array.isArray(key)) continue;++ if (key[0] === "links") {+ if (Array.isArray(data.pages)) hasLinkSource = true;+ for (const page of data.pages ?? []) {+ for (const link of page?.links ?? []) add(link);+ }+ } else if (key[0] === "dashboardData") {+ if (Array.isArray(data.links)) hasLinkSource = true;+ for (const link of data.links ?? []) add(link);+ } else if (key[0] === "link") {+ if (Number.isFinite(Number(data?.id))) hasLinkSource = true;+ add(data);+ }+ }++ return {+ links: Array.from(linksById.values()),+ hasLinkSource,+ };+};++const getFormatForCacheFile = (+ dirName: string,+ extension: string+): CacheFormat | undefined => {+ const entry = Object.entries(formatDirectories).find(+ ([, value]) => value.dirName === dirName && value.extension === extension+ );+ return entry?.[0] as CacheFormat | undefined;+};++const parseCacheFile = (+ dirName: string,+ fileName: string+): Pick<CachedFile, "linkId" | "format" | "partial"> => {+ const match = fileName.match(/^link_(\d+)\.([a-z0-9]+)(\.part)?$/i);+ if (!match) return { partial: false };++ const linkId = Number(match[1]);+ if (!Number.isFinite(linkId)) return { partial: false };++ return {+ linkId,+ format: getFormatForCacheFile(dirName, match[2].toLowerCase()),+ partial: Boolean(match[3]),+ };+};++const createEmptyCacheSnapshot = (): CacheSnapshot => ({+ bytesUsed: 0,+ filesByLinkId: new Map(),+ formatsByLinkId: new Map(),+ partialFiles: [],+});++const addFileToSnapshot = (snapshot: CacheSnapshot, file: CachedFile) => {+ snapshot.bytesUsed += file.size;++ if (file.partial) {+ snapshot.partialFiles.push(file);+ }++ if (file.linkId === undefined) return;++ const files = snapshot.filesByLinkId.get(file.linkId) ?? [];+ files.push(file);+ snapshot.filesByLinkId.set(file.linkId, files);++ if (!file.format || file.partial) return;++ const formats = snapshot.formatsByLinkId.get(file.linkId) ?? new Map();+ formats.set(file.format, file);+ snapshot.formatsByLinkId.set(file.linkId, formats);+};++const scanCacheSnapshot = async (): Promise<CacheSnapshot> => {+ const snapshot = createEmptyCacheSnapshot();+ const rootInfo = await FileSystem.getInfoAsync(archiveRoot).catch(() => null);+ if (!rootInfo?.exists) return snapshot;++ const subdirs = await FileSystem.readDirectoryAsync(archiveRoot).catch(+ () => []+ );++ await Promise.all(+ subdirs.map(async (dirName) => {+ const dirPath = `${archiveRoot}/${dirName}`;+ const files = await FileSystem.readDirectoryAsync(dirPath).catch(+ () => []+ );++ await Promise.all(+ files.map(async (fileName) => {+ const path = `${dirPath}/${fileName}`;+ const info = await FileSystem.getInfoAsync(path).catch(() => null);+ if (!info?.exists) return;++ addFileToSnapshot(snapshot, {+ path,+ size: (info as any).size ?? 0,+ modifiedAt: getFileModifiedAt(info),+ ...parseCacheFile(dirName, fileName),+ });+ })+ );+ })+ );++ return snapshot;+};++const deleteCacheFiles = async (files: Iterable<CachedFile>) => {+ let removed = 0;+ await Promise.all(+ Array.from(files).map(async (file) => {+ removed += file.size;+ await FileSystem.deleteAsync(file.path, { idempotent: true }).catch(+ () => {}+ );+ })+ );+ return removed;+};++const cleanupUnexpectedFiles = async (+ snapshot: CacheSnapshot,+ expectedFormatsByLinkId: Map<number, Set<CacheFormat>>,+ canDeleteOrphans: boolean+) => {+ const filesToDelete = new Map<string, CachedFile>();++ for (const file of snapshot.partialFiles) {+ filesToDelete.set(file.path, file);+ }++ for (const [linkId, files] of snapshot.filesByLinkId) {+ const expectedFormats = expectedFormatsByLinkId.get(linkId);++ if (!expectedFormats) {+ if (canDeleteOrphans) {+ for (const file of files) filesToDelete.set(file.path, file);+ }+ continue;+ }++ for (const file of files) {+ if (file.format && !expectedFormats.has(file.format)) {+ filesToDelete.set(file.path, file);+ }+ }+ }++ return deleteCacheFiles(filesToDelete.values());+};++const formatsForLink = (+ link: LinkIncludingShortenedCollectionAndTags+): CacheFormat[] => {+ const formats: CacheFormat[] = [];++ if (formatAvailable(link, "preview")) formats.push("preview");+ if (formatAvailable(link, "readable")) formats.push("readable");+ if (formatAvailable(link, "monolith")) formats.push("webpage");+ if (formatAvailable(link, "pdf")) formats.push("pdf");+ if (formatAvailable(link, "image")) {+ const image =+ typeof link.image === "string" ? link.image.toLowerCase() : "";+ if (image.split("?")[0].endsWith("png")) {+ formats.push("png");+ } else {+ formats.push("jpeg");+ }+ }++ return formats;+};++const missingFormatsForLink = (+ link: LinkIncludingShortenedCollectionAndTags & { id: number },+ snapshot: CacheSnapshot+): CacheFormat[] => {+ const formats = formatsForLink(link);+ const cachedFormats = snapshot.formatsByLinkId.get(link.id);+ const linkFreshnessTime = getLinkFreshnessTime(link);++ return formats.filter((format) => {+ const file = cachedFormats?.get(format);+ if (!file) return true;++ return Boolean(+ linkFreshnessTime &&+ file.modifiedAt &&+ file.modifiedAt + staleFileGraceMs < linkFreshnessTime+ );+ });+};++const downloadLinkFormats = async (+ link: LinkIncludingShortenedCollectionAndTags & { id: number },+ formats: CacheFormat[],+ auth: MobileAuth,+ userContentDomain: string | null | undefined+) => {+ const concurrency = 2;++ for (let i = 0; i < formats.length; i += concurrency) {+ if (cancelled) return;++ const slice = formats.slice(i, i + concurrency);+ await Promise.all(+ slice.map(async (format) => {+ try {+ const { delta } = await fetchFormatToCache({+ link: { id: link.id, updatedAt: link.updatedAt },+ format,+ auth,+ userContentDomain,+ });+ useOfflineSyncStore.getState().addBytes(delta);+ } catch (e) {+ console.warn(+ `[offlineSync] Failed to cache link ${link.id} format ${format}`,+ e+ );+ useOfflineSyncStore.getState().incrementFailed();+ }+ })+ );+ }+};++export const startSync = async (+ auth: MobileAuth,+ userContentDomain: string | null | undefined+) => {+ const generation = ++syncGeneration;+ activeAuth = auth;+ activeUserContentDomain = userContentDomain;++ if (runPromise) {+ cancelled = true;+ await runPromise;+ if (generation !== syncGeneration) return;+ }++ const net = await NetInfo.fetch();+ if (generation !== syncGeneration) return;+ if (!isOnline(net)) {+ useOfflineSyncStore.getState().setStatus("paused");+ return;+ }++ cancelled = false;++ const store = useOfflineSyncStore.getState();+ store.setStatus("syncing");++ const currentRun = (async () => {+ try {+ const { links, hasLinkSource } = getLinksFromCache();+ const cacheSnapshot = await scanCacheSnapshot();+ const expectedFormatsByLinkId = new Map<number, Set<CacheFormat>>();++ for (const link of links) {+ if (typeof link.id !== "number") continue;+ expectedFormatsByLinkId.set(link.id, new Set(formatsForLink(link)));+ }++ const removedBytes = await cleanupUnexpectedFiles(+ cacheSnapshot,+ expectedFormatsByLinkId,+ hasLinkSource+ );+ store.setBytesUsed(Math.max(0, cacheSnapshot.bytesUsed - removedBytes));++ store.setProgress(0, links.length);++ for (const link of links) {+ if (cancelled || generation !== syncGeneration) break;+ if (link.id === undefined) continue;++ const fullLink = link as LinkIncludingShortenedCollectionAndTags & {+ id: number;+ };++ setCachedLinkQueryData(fullLink);++ const missing = missingFormatsForLink(fullLink, cacheSnapshot);+ if (missing.length === 0) {+ store.incrementProcessed();+ continue;+ }++ store.setCurrentLinkId(fullLink.id);+ await downloadLinkFormats(fullLink, missing, auth, userContentDomain);+ store.incrementProcessed();+ }+ } catch (e) {+ console.error("[offlineSync] Sync failed", e);+ } finally {+ const currentStore = useOfflineSyncStore.getState();+ currentStore.setCurrentLinkId(null);+ if (generation === syncGeneration && currentStore.status !== "paused") {+ currentStore.setStatus("idle");+ }+ }+ })();++ runPromise = currentRun;++ try {+ await currentRun;+ } finally {+ if (runPromise === currentRun) runPromise = null;+ }+};++export const stopSync = () => {+ syncGeneration += 1;+ cancelled = true;+ activeAuth = null;+ activeUserContentDomain = undefined;+ const store = useOfflineSyncStore.getState();+ store.setCurrentLinkId(null);+ if (store.status !== "idle") store.setStatus("idle");+};++export const recomputeStorage = async () => {+ const bytes = await getCacheSize();+ useOfflineSyncStore.getState().setBytesUsed(bytes);+};++export const subscribeToConnectivity = () => {+ if (netUnsubscribe) return;++ netUnsubscribe = NetInfo.addEventListener((state) => {+ const store = useOfflineSyncStore.getState();+ if (!isOnline(state)) {+ if (runPromise) {+ cancelled = true;+ store.setStatus("paused");+ }+ } else {+ if (store.status === "paused" && activeAuth && !runPromise) {+ startSync(activeAuth, activeUserContentDomain);+ }+ }+ });+};++export const subscribeToQueryCache = () => {+ if (cacheUnsubscribe) return;++ cacheUnsubscribe = queryClient.getQueryCache().subscribe((event) => {+ if (syncQueryWriteDepth > 0) return;++ const key = event.query.queryKey[0];+ if (key !== "links" && key !== "dashboardData" && key !== "link") return;+ if (!activeAuth) return;++ if (cacheSyncTimeout) clearTimeout(cacheSyncTimeout);+ cacheSyncTimeout = setTimeout(() => {+ if (activeAuth) startSync(activeAuth, activeUserContentDomain);+ }, 1500);+ });+};++export const unsubscribeFromQueryCache = () => {+ cacheUnsubscribe?.();+ cacheUnsubscribe = null;+ if (cacheSyncTimeout) clearTimeout(cacheSyncTimeout);+ cacheSyncTimeout = null;+};++export const unsubscribeFromConnectivity = () => {+ netUnsubscribe?.();+ netUnsubscribe = null;+};++export const formatBytes = (bytes: number): string => {+ if (bytes < 1024) return `${bytes} B`;+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;+ if (bytes < 1024 * 1024 * 1024)+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;+ return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;+};diff --git a/apps/mobile/lib/queryPersister.ts b/apps/mobile/lib/queryPersister.tsindex 4e2a7beb..d7ec3fcf 100644--- a/apps/mobile/lib/queryPersister.ts+++ b/apps/mobile/lib/queryPersister.ts@@ -1,8 +1,46 @@import { createMMKV } from "react-native-mmkv";-import { Persister } from "@tanstack/react-query-persist-client";+import type {+ PersistedClient,+ Persister,+} from "@tanstack/react-query-persist-client";const storage = createMMKV({ id: "react-query" });+const infiniteQueryKeys = new Set(["links", "publicLinks", "tags"]);++const isInfiniteQueryKey = (queryKey: unknown) =>+ Array.isArray(queryKey) &&+ typeof queryKey[0] === "string" &&+ infiniteQueryKeys.has(queryKey[0]);++const hasInfiniteDataShape = (data: unknown) =>+ data == null ||+ (typeof data === "object" &&+ Array.isArray((data as any).pages) &&+ Array.isArray((data as any).pageParams));++export const sanitizePersistedClient = (+ client: PersistedClient+): PersistedClient => {+ const queries = client.clientState?.queries;+ if (!Array.isArray(queries)) return client;++ const sanitizedQueries = queries.filter((query) => {+ if (!isInfiniteQueryKey(query.queryKey)) return true;+ return hasInfiniteDataShape(query.state?.data);+ });++ if (sanitizedQueries.length === queries.length) return client;++ return {+ ...client,+ clientState: {+ ...client.clientState,+ queries: sanitizedQueries,+ },+ };+};+export const mmkvPersister: Persister = {persistClient: async (client) => {try {@@ -15,7 +53,7 @@ export const mmkvPersister: Persister = {restoreClient: async () => {try {const json = storage.getString("REACT_QUERY_CACHE");- return json ? JSON.parse(json) : undefined;+ return json ? sanitizePersistedClient(JSON.parse(json)) : undefined;} catch (e) {console.error("Error restoring client:", e);return undefined;diff --git a/apps/mobile/store/auth.ts b/apps/mobile/store/auth.tsindex f54f44d8..2e31ef71 100644--- a/apps/mobile/store/auth.ts+++ b/apps/mobile/store/auth.ts@@ -6,6 +6,8 @@ import { Alert } from "react-native";import { queryClient } from "@/lib/queryClient";import { mmkvPersister } from "@/lib/queryPersister";import { clearCache } from "@/lib/cache";+import useDataStore from "@/store/data";+import { useOfflineSyncStore } from "@/lib/offlineSync";type AuthStore = {auth: MobileAuth;@@ -142,6 +144,9 @@ const useAuthStore = create<AuthStore>((set) => ({await clearCache();+ useDataStore.getState().updateData({ offlineEnabled: false });+ useOfflineSyncStore.getState().reset();+set({auth: {instance: "",diff --git a/apps/mobile/store/data.ts b/apps/mobile/store/data.tsindex 3f0a0549..56660222 100644--- a/apps/mobile/store/data.ts+++ b/apps/mobile/store/data.ts@@ -18,6 +18,7 @@ const useDataStore = create<DataStore>((set, get) => ({theme: "system",preferredBrowser: "app",preferredCollection: null,+ offlineEnabled: false,},setData: async () => {const dataString = JSON.parse((await AsyncStorage.getItem("data")) || "{}");diff --git a/apps/web/components/SortDropdown.tsx b/apps/web/components/SortDropdown.tsxindex 58f49849..f5171388 100644--- a/apps/web/components/SortDropdown.tsx+++ b/apps/web/components/SortDropdown.tsx@@ -2,7 +2,7 @@ import React, { Dispatch, SetStateAction, useEffect } from "react";import { Sort } from "@linkwarden/types/global";import { TFunction } from "i18next";import useLocalSettingsStore from "@/store/localSettings";-import { resetInfiniteQueryPagination } from "@linkwarden/router/links";+import { resetInfiniteQueryPagination } from "@linkwarden/router/lib";import { useQueryClient } from "@tanstack/react-query";import {DropdownMenu,diff --git a/packages/router/config.tsx b/packages/router/config.tsxindex f949ed4b..334c80d2 100644--- a/packages/router/config.tsx+++ b/packages/router/config.tsx@@ -49,8 +49,12 @@ const isAtLeastInstanceVersion = (};const useConfig = (auth?: MobileAuth) => {+ const isMobile = auth !== undefined;+return useQuery({queryKey: ["config", auth?.instance ?? "web"],+ enabled:+ !isMobile || (auth.status === "authenticated" && Boolean(auth.instance)),queryFn: async () => {const response = await fetch((auth?.instance ? auth.instance : "") + "/api/v1/config",diff --git a/packages/router/lib.ts b/packages/router/lib.tsnew file mode 100644index 00000000..5a4b4c48--- /dev/null+++ b/packages/router/lib.ts@@ -0,0 +1,18 @@+import { QueryClient, QueryKey } from "@tanstack/react-query";++const resetInfiniteQueryPagination = async (+ queryClient: QueryClient,+ queryKey: QueryKey+) => {+ await queryClient.cancelQueries({ queryKey });+ queryClient.setQueriesData({ queryKey }, (oldData: any) => {+ if (!oldData) return undefined;+ return {+ pages: oldData.pages.slice(0, 1),+ pageParams: oldData.pageParams.slice(0, 1),+ };+ });+ await queryClient.invalidateQueries({ queryKey });+};++export { resetInfiniteQueryPagination };diff --git a/packages/router/links.tsx b/packages/router/links.tsxindex e797b298..1ee33347 100644--- a/packages/router/links.tsx+++ b/packages/router/links.tsx@@ -1121,22 +1121,6 @@ const useArchiveAction = () => {});};-const resetInfiniteQueryPagination = async (- queryClient: any,- queryKey: any-) => {- queryClient.setQueriesData({ queryKey }, (oldData: any) => {- if (!oldData) return undefined;-- return {- pages: oldData.pages.slice(0, 1),- pageParams: oldData.pageParams.slice(0, 1),- };- });-- await queryClient.invalidateQueries(queryKey);-};-export {useLinks,useAddLink,@@ -1147,6 +1131,5 @@ export {useGetLink,useBulkEditLinks,useArchiveAction,- resetInfiniteQueryPagination,useUpdateFile,};diff --git a/packages/types/global.ts b/packages/types/global.tsindex 59136e44..a09db28d 100644--- a/packages/types/global.ts+++ b/packages/types/global.ts@@ -209,6 +209,7 @@ export interface MobileData {theme: "light" | "dark" | "system";preferredBrowser: "app" | "system";preferredCollection: CollectionIncludingMembersAndLinkCount | null;+ offlineEnabled: boolean;}export interface WorkerStats {