Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
Added UI task to improve rclone performance of large media libraries.
nzbdav-devcb3bbec4baa1
diff --git a/frontend/app/clients/backend-client.server.ts b/frontend/app/clients/backend-client.server.tsindex 982e27de..e3505e85 100644--- a/frontend/app/clients/backend-client.server.ts+++ b/frontend/app/clients/backend-client.server.ts@@ -235,6 +235,17 @@ class BackendClient {return await response.json();}++ public async migrateLibrarySymlinks(): Promise<boolean> {+ let url = process.env.BACKEND_URL + '/api/migrate-library-symlinks';+ const apiKey = process.env.FRONTEND_BACKEND_API_KEY || "";+ const response = await fetch(url, { headers: { "x-api-key": apiKey } });+ if (!response.ok) {+ throw new Error(`Failed to begin migrating library symlinks.`);+ }++ return true;+ }}export const backendClient = new BackendClient();diff --git a/frontend/app/routes/settings/library/library.module.css b/frontend/app/routes/settings/library/library.module.cssnew file mode 100644index 00000000..e88efaca--- /dev/null+++ b/frontend/app/routes/settings/library/library.module.css@@ -0,0 +1,11 @@+.container {+ max-width:400px;+}++.input {+ margin-bottom: 8px;+}++.input::placeholder {+ color: rgba(255, 255, 255, 0.25);+}\ No newline at end of filediff --git a/frontend/app/routes/settings/library/library.module.css.d.ts b/frontend/app/routes/settings/library/library.module.css.d.tsnew file mode 100644index 00000000..beb62cbb--- /dev/null+++ b/frontend/app/routes/settings/library/library.module.css.d.ts@@ -0,0 +1,11 @@+declare const styles: {+ readonly "container": string;+ readonly "input": string;+ readonly "error": string;+ readonly "run": string;+ readonly "task-progress": string;+ readonly "list": string;+ readonly "list-item": string;+};+export = styles;+diff --git a/frontend/app/routes/settings/library/library.tsx b/frontend/app/routes/settings/library/library.tsxnew file mode 100644index 00000000..39c112ea--- /dev/null+++ b/frontend/app/routes/settings/library/library.tsx@@ -0,0 +1,34 @@+import { Form } from "react-bootstrap";+import styles from "./library.module.css"+import { type Dispatch, type SetStateAction } from "react";++type LibrarySettingsProps = {+ savedConfig: Record<string, string>+ config: Record<string, string>+ setNewConfig: Dispatch<SetStateAction<Record<string, string>>>+};++export function LibrarySettings({ savedConfig, config, setNewConfig }: LibrarySettingsProps) {+ return (+ <div className={styles.container}>+ <Form.Group>+ <Form.Label htmlFor="library-dir-input">Library Directory</Form.Label>+ <Form.Control+ className={styles.input}+ type="text"+ id="library-dir-input"+ aria-describedby="library-dir-help"+ value={config["media.library-dir"]}+ onChange={e => setNewConfig({ ...config, "media.library-dir": e.target.value })} />+ <Form.Text id="library-dir-help" muted>+ The path to your organized media library that contains all your imported symlinks.+ Disregard this setting if you're not upgrading from version 0.2.x.+ </Form.Text>+ </Form.Group>+ </div>+ );+}++export function isLibrarySettingsUpdated(config: Record<string, string>, newConfig: Record<string, string>) {+ return config["media.library-dir"] !== newConfig["media.library-dir"]+}\ No newline at end of filediff --git a/frontend/app/routes/settings/maintenance/maintenance.module.css b/frontend/app/routes/settings/maintenance/maintenance.module.cssnew file mode 100644index 00000000..a6539c8c--- /dev/null+++ b/frontend/app/routes/settings/maintenance/maintenance.module.css@@ -0,0 +1,31 @@+.container {+ max-width: 600px;+}++.task {+ background-color: #02181d;+ padding: 1rem;+ color: #dee2e6;+ border-radius: .375rem;+ box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.25);+}++.run {+ margin-top: 12px;+ display: flex;+ flex-direction: row;+ gap: 10px;+ align-items: center;+}++.task-progress {+ font-size:12px;+}++.list {+ margin-bottom: 0;+}++.list-item {+ margin-top: 5px+}\ No newline at end of filediff --git a/frontend/app/routes/settings/maintenance/maintenance.module.css.d.ts b/frontend/app/routes/settings/maintenance/maintenance.module.css.d.tsnew file mode 100644index 00000000..19f8fdb4--- /dev/null+++ b/frontend/app/routes/settings/maintenance/maintenance.module.css.d.ts@@ -0,0 +1,11 @@+declare const styles: {+ readonly "container": string;+ readonly "task": string;+ readonly "title": string;+ readonly "run": string;+ readonly "task-progress": string;+ readonly "list": string;+ readonly "list-item": string;+};+export = styles;+diff --git a/frontend/app/routes/settings/maintenance/maintenance.tsx b/frontend/app/routes/settings/maintenance/maintenance.tsxnew file mode 100644index 00000000..776905ed--- /dev/null+++ b/frontend/app/routes/settings/maintenance/maintenance.tsx@@ -0,0 +1,117 @@+import { Alert, Button, Form } from "react-bootstrap";+import styles from "./maintenance.module.css"+import { useCallback, useEffect, useState } from "react";++type MaintenanceProps = {+ savedConfig: Record<string, string>+};++export function Maintenance({ savedConfig }: MaintenanceProps) {+ const [connected, setConnected] = useState<boolean>(false);+ const [progress, setProgress] = useState<string | null>(null);+ const [isFetching, setIsFetching] = useState<boolean>(false);+ const libraryDir = savedConfig["media.library-dir"];+ const isFinished = progress === "complete" || progress?.startsWith("failed");+ const isRunning = !isFinished && (isFetching || progress !== null);+ const isRunButtonEnabled = !!libraryDir && connected && !isRunning;+ const runButtonVariant = isRunButtonEnabled ? 'success' : 'secondary';+ const runButtonLabel = isRunning ? "⌛ Running.." : '▶ Run Task';+ let [retargetted, processed] = ["0", "0"];+ if (isRunning && progress?.includes("/")) {+ const parts = progress.split("/");+ retargetted = parts[0];+ processed = parts[1];+ }++ useEffect(() => {+ let ws: WebSocket;+ let disposed = false;+ function connect() {+ ws = new WebSocket(window.location.origin.replace(/^http/, 'ws'));+ ws.onmessage = (event) => setProgress(event.data);+ ws.onopen = () => { setConnected(true); ws.send('stp'); }+ ws.onclose = () => { !disposed && setTimeout(() => connect(), 1000); setProgress(null) };+ ws.onerror = () => { ws.close() };+ return () => { disposed = true; ws.close(); }+ }+ return connect();+ }, [setProgress]);++ const onRun = useCallback(async () => {+ setIsFetching(true);+ await fetch("/tasks/migrate-library-symlinks");+ setIsFetching(false);+ }, [setIsFetching]);++ return (+ <div className={styles.container}>+ {!libraryDir &&+ <Alert variant="warning">+ Warning+ <ul className={styles.list}>+ <li className={styles["list-item"]}>+ You must first configure the Library Directory setting before running this task.+ Head over to the Library tab.+ </li>+ </ul>+ </Alert>+ }+ {libraryDir &&+ <Alert variant="danger">+ <span style={{fontWeight: 'bold'}}>Danger</span>+ <ul className={styles.list}>+ <li className={styles["list-item"]}>+ Make a backup of your organized media library+ symlinks before running this task+ </li>+ <li className={styles["list-item"]}>+ Symlinks will be deleted and recreated with the+ same PUID/PGID that the NzbDAV process is running+ under+ </li>+ </ul>+ </Alert>+ }+ <div className={styles.task}>+ <Form.Group>+ <Form.Label className={styles.title}>Update Symlink Targets</Form.Label>+ <div className={styles.run}>+ <Button variant={runButtonVariant} onClick={onRun} disabled={!isRunButtonEnabled}>+ {runButtonLabel}+ </Button>+ {isRunning &&+ <div className={styles["task-progress"]}>+ Processed: {processed} <br />+ Re-targetted: {retargetted}+ </div>+ }+ {isFinished &&+ <div className={styles["task-progress"]}>+ {progress}+ </div>+ }+ </div>+ <Form.Text id="symlink-task-progress-help" muted>+ <br />+ Prior to version 0.3.x, all symlink targets would point to the `/content`+ folder within the webdav. This caused performance issues with RClone since+ `/content/tv` often contained thousands of children, which RClone+ handles <a href="https://github.com/rclone/rclone/issues/8759">inefficiently</a>.+ <br /><br />+ Symlinks in version 0.3.x and onward now point to the `/.ids` folder within the+ webdav, which ensures that no single directory contains any more than a few+ children. This improves RClone performance and allows the library to scale better.+ <br /><br />+ This task recreates previously imported symlinks and re-targets them to the+ `/.ids` folder. It only needs to happen once. This task will be deprecated and+ removed when we reach 1.0 release. Be sure to run this task and rebuild your+ symlinks by then. If you've never used version 0.2.x, then there is no need to+ run this task. Those who have only used version 0.3.x onward should be good+ from the start.+ </Form.Text>+ </Form.Group>+ </div>+ </div>++ );+}\ No newline at end of filediff --git a/frontend/app/routes/settings/route.module.css b/frontend/app/routes/settings/route.module.cssindex 23ad78a5..e1b207e2 100644--- a/frontend/app/routes/settings/route.module.css+++ b/frontend/app/routes/settings/route.module.css@@ -11,6 +11,15 @@--bs-nav-link-color: #93a1a1;--bs-nav-link-hover-color: #93a1a1;margin-bottom: 25px;+ overflow-x: auto;+ overflow-y: hidden;+ flex-wrap: unset;+ scrollbar-width: none;+ white-space: nowrap;+}++.tabs::-webkit-scrollbar {+ display: none;}.button {diff --git a/frontend/app/routes/settings/route.tsx b/frontend/app/routes/settings/route.tsxindex 1389cdf9..e8512933 100644--- a/frontend/app/routes/settings/route.tsx+++ b/frontend/app/routes/settings/route.tsx@@ -1,11 +1,13 @@import type { Route } from "./+types/route";import styles from "./route.module.css"-import { Tabs, Tab, Button } from "react-bootstrap"+import { Tabs, Tab, Button, Form } from "react-bootstrap"import { backendClient } from "~/clients/backend-client.server";import { isUsenetSettingsUpdated, UsenetSettings } from "./usenet/usenet";-import React from "react";+import React, { useEffect } from "react";import { isSabnzbdSettingsUpdated, isSabnzbdSettingsValid, SabnzbdSettings } from "./sabnzbd/sabnzbd";import { isWebdavSettingsUpdated, isWebdavSettingsValid, WebdavSettings } from "./webdav/webdav";+import { isLibrarySettingsUpdated, LibrarySettings } from "./library/library";+import { Maintenance } from "./maintenance/maintenance";const defaultConfig = {"api.key": "",@@ -22,8 +24,11 @@ const defaultConfig = {"webdav.pass": "","webdav.show-hidden-files": "false","rclone.mount-dir": "",+ "media.library-dir": "",}+const advancedTabs = ["library", "maintenance"];+export async function loader({ request }: Route.LoaderArgs) {// fetch the config itemsvar configItems = await backendClient.getConfig(Object.keys(defaultConfig));@@ -47,20 +52,26 @@ type BodyProps = {};function Body(props: BodyProps) {+ // stateful variablesconst [config, setConfig] = React.useState(props.config);const [newConfig, setNewConfig] = React.useState(config);const [isUsenetSettingsReadyToSave, setIsUsenetSettingsReadyToSave] = React.useState(false);const [isSaving, setIsSaving] = React.useState(false);const [isSaved, setIsSaved] = React.useState(false);+ const [showAdvanced, setShowAdvanced] = React.useState(false);+ const [activeTab, setActiveTab] = React.useState('usenet');+ // derived variablesconst iseUsenetUpdated = isUsenetSettingsUpdated(config, newConfig);const isSabnzbdUpdated = isSabnzbdSettingsUpdated(config, newConfig);const isWebdavUpdated = isWebdavSettingsUpdated(config, newConfig);- const isUpdated = iseUsenetUpdated || isSabnzbdUpdated || isWebdavUpdated;+ const isLibraryUpdated = isLibrarySettingsUpdated(config, newConfig);+ const isUpdated = iseUsenetUpdated || isSabnzbdUpdated || isWebdavUpdated || isLibraryUpdated;const usenetTitle = iseUsenetUpdated ? "Usenet ✏️" : "Usenet";const sabnzbdTitle = isSabnzbdUpdated ? "SABnzbd ✏️" : "SABnzbd";const webdavTitle = isWebdavUpdated ? "WebDAV ✏️" : "WebDAV";+ const libraryTitle = isLibraryUpdated ? "Library ✏️" : "Library";const saveButtonLabel = isSaving ? "Saving...": !isUpdated && isSaved ? "Saved ✅"@@ -74,6 +85,13 @@ function Body(props: BodyProps) {: "secondary";const isSaveButtonDisabled = saveButtonLabel !== "Save";+ // effects+ useEffect(() => {+ if (!showAdvanced && advancedTabs.includes(activeTab)) {+ setActiveTab("usenet");+ }+ }, [showAdvanced, activeTab, setActiveTab])+// eventsconst onClear = React.useCallback(() => {setNewConfig(config);@@ -106,7 +124,8 @@ function Body(props: BodyProps) {return (<div className={styles.container}><Tabs- defaultActiveKey="usenet"+ activeKey={activeTab}+ onSelect={(k) => setActiveTab(k!)}className={styles.tabs}><Tab eventKey="usenet" title={usenetTitle}>@@ -118,8 +137,27 @@ function Body(props: BodyProps) {<Tab eventKey="webdav" title={webdavTitle}><WebdavSettings config={newConfig} setNewConfig={setNewConfig} /></Tab>+ {showAdvanced &&+ <Tab eventKey="library" title={libraryTitle}>+ <LibrarySettings savedConfig={config} config={newConfig} setNewConfig={setNewConfig} />+ </Tab>+ }+ {showAdvanced &&+ <Tab eventKey="maintenance" title="Maintenance">+ <Maintenance savedConfig={config} />+ </Tab>+ }</Tabs><hr />++ <Form.Check+ style={{ marginBottom: '20px' }}+ type="checkbox"+ id="advanced-settings-checkbox"+ label="Show Advanced Settings"+ checked={showAdvanced}+ onChange={(e) => setShowAdvanced(Boolean(e.target.checked))} />+{isUpdated && <ButtonclassName={styles.button}variant="secondary"diff --git a/frontend/app/routes/tasks.migrate-library-symlinks/route.tsx b/frontend/app/routes/tasks.migrate-library-symlinks/route.tsxnew file mode 100644index 00000000..b0fcb147--- /dev/null+++ b/frontend/app/routes/tasks.migrate-library-symlinks/route.tsx@@ -0,0 +1,13 @@+import type { Route } from "./+types/route";+import { backendClient } from "~/clients/backend-client.server";+import { redirect } from "react-router";+import { sessionStorage } from "~/auth/authentication.server";++export async function loader({ request }: Route.ActionArgs) {+ // ensure user is logged in+ let session = await sessionStorage.getSession(request.headers.get("cookie"));+ let user = session.get("user");+ if (!user) return redirect("/login");++ return await backendClient.migrateLibrarySymlinks();+}\ No newline at end of file