Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#

Added UI task to improve rclone performance of large media libraries.

nzbdav-devcb3bbec4baa1
diff --git a/frontend/app/clients/backend-client.server.ts b/frontend/app/clients/backend-client.server.ts
index 982e27de..e3505e85 100644
--- a/frontend/app/clients/backend-client.server.ts
+++ b/frontend/app/clients/backend-client.server.ts
@@ -235,6 +235,17 @@ class BackendClient {
return await response.json();
}
+
+ public async migrateLibrarySymlinks(): Promise<boolean> {
+ let url = process.env.BACKEND_URL + '/api/migrate-library-symlinks';
+ const apiKey = process.env.FRONTEND_BACKEND_API_KEY || "";
+ const response = await fetch(url, { headers: { "x-api-key": apiKey } });
+ if (!response.ok) {
+ throw new Error(`Failed to begin migrating library symlinks.`);
+ }
+
+ return true;
+ }
}
export const backendClient = new BackendClient();
diff --git a/frontend/app/routes/settings/library/library.module.css b/frontend/app/routes/settings/library/library.module.css
new file mode 100644
index 00000000..e88efaca
--- /dev/null
+++ b/frontend/app/routes/settings/library/library.module.css
@@ -0,0 +1,11 @@
+.container {
+ max-width:400px;
+}
+
+.input {
+ margin-bottom: 8px;
+}
+
+.input::placeholder {
+ color: rgba(255, 255, 255, 0.25);
+}
\ No newline at end of file
diff --git a/frontend/app/routes/settings/library/library.module.css.d.ts b/frontend/app/routes/settings/library/library.module.css.d.ts
new file mode 100644
index 00000000..beb62cbb
--- /dev/null
+++ b/frontend/app/routes/settings/library/library.module.css.d.ts
@@ -0,0 +1,11 @@
+declare const styles: {
+ readonly "container": string;
+ readonly "input": string;
+ readonly "error": string;
+ readonly "run": string;
+ readonly "task-progress": string;
+ readonly "list": string;
+ readonly "list-item": string;
+};
+export = styles;
+
diff --git a/frontend/app/routes/settings/library/library.tsx b/frontend/app/routes/settings/library/library.tsx
new file mode 100644
index 00000000..39c112ea
--- /dev/null
+++ b/frontend/app/routes/settings/library/library.tsx
@@ -0,0 +1,34 @@
+import { Form } from "react-bootstrap";
+import styles from "./library.module.css"
+import { type Dispatch, type SetStateAction } from "react";
+
+type LibrarySettingsProps = {
+ savedConfig: Record<string, string>
+ config: Record<string, string>
+ setNewConfig: Dispatch<SetStateAction<Record<string, string>>>
+};
+
+export function LibrarySettings({ savedConfig, config, setNewConfig }: LibrarySettingsProps) {
+ return (
+ <div className={styles.container}>
+ <Form.Group>
+ <Form.Label htmlFor="library-dir-input">Library Directory</Form.Label>
+ <Form.Control
+ className={styles.input}
+ type="text"
+ id="library-dir-input"
+ aria-describedby="library-dir-help"
+ value={config["media.library-dir"]}
+ onChange={e => setNewConfig({ ...config, "media.library-dir": e.target.value })} />
+ <Form.Text id="library-dir-help" muted>
+ The path to your organized media library that contains all your imported symlinks.
+ Disregard this setting if you're not upgrading from version 0.2.x.
+ </Form.Text>
+ </Form.Group>
+ </div>
+ );
+}
+
+export function isLibrarySettingsUpdated(config: Record<string, string>, newConfig: Record<string, string>) {
+ return config["media.library-dir"] !== newConfig["media.library-dir"]
+}
\ No newline at end of file
diff --git a/frontend/app/routes/settings/maintenance/maintenance.module.css b/frontend/app/routes/settings/maintenance/maintenance.module.css
new file mode 100644
index 00000000..a6539c8c
--- /dev/null
+++ b/frontend/app/routes/settings/maintenance/maintenance.module.css
@@ -0,0 +1,31 @@
+.container {
+ max-width: 600px;
+}
+
+.task {
+ background-color: #02181d;
+ padding: 1rem;
+ color: #dee2e6;
+ border-radius: .375rem;
+ box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.25);
+}
+
+.run {
+ margin-top: 12px;
+ display: flex;
+ flex-direction: row;
+ gap: 10px;
+ align-items: center;
+}
+
+.task-progress {
+ font-size:12px;
+}
+
+.list {
+ margin-bottom: 0;
+}
+
+.list-item {
+ margin-top: 5px
+}
\ No newline at end of file
diff --git a/frontend/app/routes/settings/maintenance/maintenance.module.css.d.ts b/frontend/app/routes/settings/maintenance/maintenance.module.css.d.ts
new file mode 100644
index 00000000..19f8fdb4
--- /dev/null
+++ b/frontend/app/routes/settings/maintenance/maintenance.module.css.d.ts
@@ -0,0 +1,11 @@
+declare const styles: {
+ readonly "container": string;
+ readonly "task": string;
+ readonly "title": string;
+ readonly "run": string;
+ readonly "task-progress": string;
+ readonly "list": string;
+ readonly "list-item": string;
+};
+export = styles;
+
diff --git a/frontend/app/routes/settings/maintenance/maintenance.tsx b/frontend/app/routes/settings/maintenance/maintenance.tsx
new file mode 100644
index 00000000..776905ed
--- /dev/null
+++ b/frontend/app/routes/settings/maintenance/maintenance.tsx
@@ -0,0 +1,117 @@
+import { Alert, Button, Form } from "react-bootstrap";
+import styles from "./maintenance.module.css"
+import { useCallback, useEffect, useState } from "react";
+
+type MaintenanceProps = {
+ savedConfig: Record<string, string>
+};
+
+export function Maintenance({ savedConfig }: MaintenanceProps) {
+ const [connected, setConnected] = useState<boolean>(false);
+ const [progress, setProgress] = useState<string | null>(null);
+ const [isFetching, setIsFetching] = useState<boolean>(false);
+ const libraryDir = savedConfig["media.library-dir"];
+ const isFinished = progress === "complete" || progress?.startsWith("failed");
+ const isRunning = !isFinished && (isFetching || progress !== null);
+ const isRunButtonEnabled = !!libraryDir && connected && !isRunning;
+ const runButtonVariant = isRunButtonEnabled ? 'success' : 'secondary';
+ const runButtonLabel = isRunning ? "⌛ Running.." : '▶ Run Task';
+ let [retargetted, processed] = ["0", "0"];
+ if (isRunning && progress?.includes("/")) {
+ const parts = progress.split("/");
+ retargetted = parts[0];
+ processed = parts[1];
+ }
+
+ useEffect(() => {
+ let ws: WebSocket;
+ let disposed = false;
+ function connect() {
+ ws = new WebSocket(window.location.origin.replace(/^http/, 'ws'));
+ ws.onmessage = (event) => setProgress(event.data);
+ ws.onopen = () => { setConnected(true); ws.send('stp'); }
+ ws.onclose = () => { !disposed && setTimeout(() => connect(), 1000); setProgress(null) };
+ ws.onerror = () => { ws.close() };
+ return () => { disposed = true; ws.close(); }
+ }
+ return connect();
+ }, [setProgress]);
+
+ const onRun = useCallback(async () => {
+ setIsFetching(true);
+ await fetch("/tasks/migrate-library-symlinks");
+ setIsFetching(false);
+ }, [setIsFetching]);
+
+ return (
+ <div className={styles.container}>
+ {!libraryDir &&
+ <Alert variant="warning">
+ Warning
+ <ul className={styles.list}>
+ <li className={styles["list-item"]}>
+ You must first configure the Library Directory setting before running this task.
+ Head over to the Library tab.
+ </li>
+ </ul>
+ </Alert>
+ }
+ {libraryDir &&
+ <Alert variant="danger">
+ <span style={{fontWeight: 'bold'}}>Danger</span>
+ <ul className={styles.list}>
+ <li className={styles["list-item"]}>
+ Make a backup of your organized media library
+ symlinks before running this task
+ </li>
+ <li className={styles["list-item"]}>
+ Symlinks will be deleted and recreated with the
+ same PUID/PGID that the NzbDAV process is running
+ under
+ </li>
+ </ul>
+ </Alert>
+ }
+ <div className={styles.task}>
+ <Form.Group>
+ <Form.Label className={styles.title}>Update Symlink Targets</Form.Label>
+ <div className={styles.run}>
+ <Button variant={runButtonVariant} onClick={onRun} disabled={!isRunButtonEnabled}>
+ {runButtonLabel}
+ </Button>
+ {isRunning &&
+ <div className={styles["task-progress"]}>
+ Processed: {processed} <br />
+ Re-targetted: {retargetted}
+ </div>
+ }
+ {isFinished &&
+ <div className={styles["task-progress"]}>
+ {progress}
+ </div>
+ }
+ </div>
+ <Form.Text id="symlink-task-progress-help" muted>
+ <br />
+ Prior to version 0.3.x, all symlink targets would point to the `/content`
+ folder within the webdav. This caused performance issues with RClone since
+ `/content/tv` often contained thousands of children, which RClone
+ handles <a href="https://github.com/rclone/rclone/issues/8759">inefficiently</a>.
+ <br /><br />
+ Symlinks in version 0.3.x and onward now point to the `/.ids` folder within the
+ webdav, which ensures that no single directory contains any more than a few
+ children. This improves RClone performance and allows the library to scale better.
+ <br /><br />
+ This task recreates previously imported symlinks and re-targets them to the
+ `/.ids` folder. It only needs to happen once. This task will be deprecated and
+ removed when we reach 1.0 release. Be sure to run this task and rebuild your
+ symlinks by then. If you've never used version 0.2.x, then there is no need to
+ run this task. Those who have only used version 0.3.x onward should be good
+ from the start.
+ </Form.Text>
+ </Form.Group>
+ </div>
+ </div>
+
+ );
+}
\ No newline at end of file
diff --git a/frontend/app/routes/settings/route.module.css b/frontend/app/routes/settings/route.module.css
index 23ad78a5..e1b207e2 100644
--- a/frontend/app/routes/settings/route.module.css
+++ b/frontend/app/routes/settings/route.module.css
@@ -11,6 +11,15 @@
--bs-nav-link-color: #93a1a1;
--bs-nav-link-hover-color: #93a1a1;
margin-bottom: 25px;
+ overflow-x: auto;
+ overflow-y: hidden;
+ flex-wrap: unset;
+ scrollbar-width: none;
+ white-space: nowrap;
+}
+
+.tabs::-webkit-scrollbar {
+ display: none;
}
.button {
diff --git a/frontend/app/routes/settings/route.tsx b/frontend/app/routes/settings/route.tsx
index 1389cdf9..e8512933 100644
--- a/frontend/app/routes/settings/route.tsx
+++ b/frontend/app/routes/settings/route.tsx
@@ -1,11 +1,13 @@
import type { Route } from "./+types/route";
import styles from "./route.module.css"
-import { Tabs, Tab, Button } from "react-bootstrap"
+import { Tabs, Tab, Button, Form } from "react-bootstrap"
import { backendClient } from "~/clients/backend-client.server";
import { isUsenetSettingsUpdated, UsenetSettings } from "./usenet/usenet";
-import React from "react";
+import React, { useEffect } from "react";
import { isSabnzbdSettingsUpdated, isSabnzbdSettingsValid, SabnzbdSettings } from "./sabnzbd/sabnzbd";
import { isWebdavSettingsUpdated, isWebdavSettingsValid, WebdavSettings } from "./webdav/webdav";
+import { isLibrarySettingsUpdated, LibrarySettings } from "./library/library";
+import { Maintenance } from "./maintenance/maintenance";
const defaultConfig = {
"api.key": "",
@@ -22,8 +24,11 @@ const defaultConfig = {
"webdav.pass": "",
"webdav.show-hidden-files": "false",
"rclone.mount-dir": "",
+ "media.library-dir": "",
}
+const advancedTabs = ["library", "maintenance"];
+
export async function loader({ request }: Route.LoaderArgs) {
// fetch the config items
var configItems = await backendClient.getConfig(Object.keys(defaultConfig));
@@ -47,20 +52,26 @@ type BodyProps = {
};
function Body(props: BodyProps) {
+ // stateful variables
const [config, setConfig] = React.useState(props.config);
const [newConfig, setNewConfig] = React.useState(config);
const [isUsenetSettingsReadyToSave, setIsUsenetSettingsReadyToSave] = React.useState(false);
const [isSaving, setIsSaving] = React.useState(false);
const [isSaved, setIsSaved] = React.useState(false);
+ const [showAdvanced, setShowAdvanced] = React.useState(false);
+ const [activeTab, setActiveTab] = React.useState('usenet');
+ // derived variables
const iseUsenetUpdated = isUsenetSettingsUpdated(config, newConfig);
const isSabnzbdUpdated = isSabnzbdSettingsUpdated(config, newConfig);
const isWebdavUpdated = isWebdavSettingsUpdated(config, newConfig);
- const isUpdated = iseUsenetUpdated || isSabnzbdUpdated || isWebdavUpdated;
+ const isLibraryUpdated = isLibrarySettingsUpdated(config, newConfig);
+ const isUpdated = iseUsenetUpdated || isSabnzbdUpdated || isWebdavUpdated || isLibraryUpdated;
const usenetTitle = iseUsenetUpdated ? "Usenet ✏️" : "Usenet";
const sabnzbdTitle = isSabnzbdUpdated ? "SABnzbd ✏️" : "SABnzbd";
const webdavTitle = isWebdavUpdated ? "WebDAV ✏️" : "WebDAV";
+ const libraryTitle = isLibraryUpdated ? "Library ✏️" : "Library";
const saveButtonLabel = isSaving ? "Saving..."
: !isUpdated && isSaved ? "Saved ✅"
@@ -74,6 +85,13 @@ function Body(props: BodyProps) {
: "secondary";
const isSaveButtonDisabled = saveButtonLabel !== "Save";
+ // effects
+ useEffect(() => {
+ if (!showAdvanced && advancedTabs.includes(activeTab)) {
+ setActiveTab("usenet");
+ }
+ }, [showAdvanced, activeTab, setActiveTab])
+
// events
const onClear = React.useCallback(() => {
setNewConfig(config);
@@ -106,7 +124,8 @@ function Body(props: BodyProps) {
return (
<div className={styles.container}>
<Tabs
- defaultActiveKey="usenet"
+ activeKey={activeTab}
+ onSelect={(k) => setActiveTab(k!)}
className={styles.tabs}
>
<Tab eventKey="usenet" title={usenetTitle}>
@@ -118,8 +137,27 @@ function Body(props: BodyProps) {
<Tab eventKey="webdav" title={webdavTitle}>
<WebdavSettings config={newConfig} setNewConfig={setNewConfig} />
</Tab>
+ {showAdvanced &&
+ <Tab eventKey="library" title={libraryTitle}>
+ <LibrarySettings savedConfig={config} config={newConfig} setNewConfig={setNewConfig} />
+ </Tab>
+ }
+ {showAdvanced &&
+ <Tab eventKey="maintenance" title="Maintenance">
+ <Maintenance savedConfig={config} />
+ </Tab>
+ }
</Tabs>
<hr />
+
+ <Form.Check
+ style={{ marginBottom: '20px' }}
+ type="checkbox"
+ id="advanced-settings-checkbox"
+ label="Show Advanced Settings"
+ checked={showAdvanced}
+ onChange={(e) => setShowAdvanced(Boolean(e.target.checked))} />
+
{isUpdated && <Button
className={styles.button}
variant="secondary"
diff --git a/frontend/app/routes/tasks.migrate-library-symlinks/route.tsx b/frontend/app/routes/tasks.migrate-library-symlinks/route.tsx
new file mode 100644
index 00000000..b0fcb147
--- /dev/null
+++ b/frontend/app/routes/tasks.migrate-library-symlinks/route.tsx
@@ -0,0 +1,13 @@
+import type { Route } from "./+types/route";
+import { backendClient } from "~/clients/backend-client.server";
+import { redirect } from "react-router";
+import { sessionStorage } from "~/auth/authentication.server";
+
+export async function loader({ request }: Route.ActionArgs) {
+ // ensure user is logged in
+ let session = await sessionStorage.getSession(request.headers.get("cookie"));
+ let user = session.get("user");
+ if (!user) return redirect("/login");
+
+ return await backendClient.migrateLibrarySymlinks();
+}
\ No newline at end of file