best way to save what you love
Svelte collaborationdownloaderinstagramjavascriptmusicredditsocial-mediasoundcloudsveltetiktoktwittertypescriptvideovimeovkwebappyoutubeyoutube-downloader
api: dynamic env reloading from path/url
jjba2d266de79a
diff --git a/api/src/config.js b/api/src/config.jsindex e7e29652..a5ebf3d6 100644--- a/api/src/config.js+++ b/api/src/config.js@@ -1,18 +1,34 @@import { getVersion } from "@imput/version-info";-import { loadEnvs, validateEnvs } from "./core/env.js";+import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js";+import * as cluster from "./misc/cluster.js";const version = await getVersion();-let env = loadEnvs();+const env = loadEnvs();const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;+export const canonicalEnv = Object.freeze(structuredClone(process.env));export const setTunnelPort = (port) => env.tunnelPort = port;export const isCluster = env.instanceCount > 1;+export const updateEnv = (newEnv) => {+ // tunnelPort is special and needs to get carried over here+ newEnv.tunnelPort = env.tunnelPort;++ for (const key in env) {+ env[key] = newEnv[key];+ }++ cluster.broadcast({ env_update: newEnv });+}await validateEnvs(env);+if (env.envFile) {+ setupEnvWatcher();+}+export {env,genericUserAgent,diff --git a/api/src/core/api.js b/api/src/core/api.jsindex d8ed8ccd..7c0e15b9 100644--- a/api/src/core/api.js+++ b/api/src/core/api.js@@ -48,19 +48,23 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {const startTime = new Date();const startTimestamp = startTime.getTime();- const serverInfo = JSON.stringify({- cobalt: {- version: version,- url: env.apiURL,- startTime: `${startTimestamp}`,- durationLimit: env.durationLimit,- turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,- services: [...env.enabledServices].map(e => {- return friendlyServiceName(e);- }),- },- git,- })+ const getServerInfo = () => {+ return JSON.stringify({+ cobalt: {+ version: version,+ url: env.apiURL,+ startTime: `${startTimestamp}`,+ durationLimit: env.durationLimit,+ turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,+ services: [...env.enabledServices].map(e => {+ return friendlyServiceName(e);+ }),+ },+ git,+ });+ }++ const serverInfo = getServerInfo();const handleRateExceeded = (_, res) => {const { body } = createResponse("error", {@@ -311,7 +315,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {app.get('/', (_, res) => {res.type('json');- res.status(200).send(serverInfo);+ res.status(200).send(env.envFile ? getServerInfo() : serverInfo);})app.get('/favicon.ico', (req, res) => {@@ -331,10 +335,6 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutesif (env.externalProxy) {- if (env.freebindCIDR) {- throw new Error('Freebind is not available when external proxy is enabled')- }-setGlobalDispatcher(new ProxyAgent(env.externalProxy))}diff --git a/api/src/core/env.js b/api/src/core/env.jsindex 34a0d6ef..b50afd1c 100644--- a/api/src/core/env.js+++ b/api/src/core/env.js@@ -1,6 +1,11 @@import { Constants } from "youtubei.js";-import { supportsReusePort } from "../misc/cluster.js";import { services } from "../processing/service-config.js";+import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";++import { FileWatcher } from "../misc/file-watcher.js";+import { isURL } from "../misc/utils.js";+import * as cluster from "../misc/cluster.js";+import { Yellow } from "../misc/console-text.js";const forceLocalProcessingOptions = ["never", "session", "always"];@@ -68,6 +73,9 @@ export const loadEnvs = (env = process.env) => {// "never" | "session" | "always"forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never",++ envFile: env.API_ENV_FILE,+ envRemoteReloadInterval: 300,};}@@ -78,7 +86,7 @@ export const validateEnvs = async (env) => {if (env.instanceCount > 1 && !env.redisURL) {throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");- } else if (env.instanceCount > 1 && !await supportsReusePort()) {+ } else if (env.instanceCount > 1 && !await cluster.supportsReusePort()) {console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');console.error('(or other OS that supports it). for more info, see `reusePort` option on');@@ -97,4 +105,81 @@ export const validateEnvs = async (env) => {console.error(`Supported options are are: ${forceLocalProcessingOptions.join(', ')}\n`);throw new Error("Invalid FORCE_LOCAL_PROCESSING");}++ if (env.externalProxy && env.freebindCIDR) {+ throw new Error('freebind is not available when external proxy is enabled')+ }+}++const reloadEnvs = async (contents) => {+ const newEnvs = {};++ for (let line of (await contents).split('\n')) {+ line = line.trim();+ if (line === '') {+ continue;+ }++ const [ key, value ] = line.split(/=(.+)?/);+ if (key) {+ newEnvs[key] = value || '';+ }+ }++ const candidate = {+ ...canonicalEnv,+ ...newEnvs,+ };++ const parsed = loadEnvs(candidate);+ await validateEnvs(parsed);+ updateEnv(parsed);+}++const wrapReload = (contents) => {+ reloadEnvs(contents)+ .catch((e) => {+ console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`);+ console.error('Error:', e);+ });+}++let watcher;+const setupWatcherFromFile = (path) => {+ const load = () => wrapReload(watcher.read());++ if (isURL(path)) {+ watcher = FileWatcher.fromFileProtocol(path);+ } else {+ watcher = new FileWatcher({ path });+ }++ watcher.on('file-updated', load);+ load();+}++const setupWatcherFromFetch = (url) => {+ const load = () => wrapReload(fetch(url).then(r => r.text()));+ setInterval(load, currentEnv.envRemoteReloadInterval);+ load();+}++export const setupEnvWatcher = () => {+ if (cluster.isPrimary) {+ const envFile = currentEnv.envFile;+ const isFile = !isURL(envFile)+ || new URL(envFile).protocol === 'file:';++ if (isFile) {+ setupWatcherFromFile(envFile);+ } else {+ setupWatcherFromFetch(envFile);+ }+ } else if (cluster.isWorker) {+ process.on('message', (message) => {+ if ('env_update' in message) {+ updateEnv(message.env_update);+ }+ });+ }}diff --git a/api/src/misc/utils.js b/api/src/misc/utils.jsindex 62bf6351..1cde3cdc 100644--- a/api/src/misc/utils.js+++ b/api/src/misc/utils.js@@ -52,3 +52,12 @@ export function splitFilenameExtension(filename) {export function zip(a, b) {return a.map((value, i) => [ value, b[i] ]);}++export function isURL(input) {+ try {+ new URL(input);+ return true;+ } catch {+ return false;+ }+}diff --git a/docs/api-env-variables.md b/docs/api-env-variables.mdindex d44e1cab..92e7a6cf 100644--- a/docs/api-env-variables.md+++ b/docs/api-env-variables.md@@ -13,6 +13,7 @@ this document is not final and will expand over time. feel free to improve it!| API_REDIS_URL | | `redis://localhost:6379` || DISABLED_SERVICES | | `bilibili,youtube` || FORCE_LOCAL_PROCESSING | `never` | `always` |+| API_ENV_FILE | | `/.env` |[*view details*](#general)@@ -111,6 +112,9 @@ when set to `session`, only requests from session (Bearer token) clients will bewhen set to `always`, all requests will be forced to use on-device processing, no matter the preference.+### API_ENV_FILE+the URL or local path to a `key=value`-style environment variable file. this is used for dynamically reloading environment variables. **not all environment variables are able to be updated by this.** (e.g. the ratelimiters are instantiated when starting cobalt, and cannot be changed)+## networking[*jump to the table*](#networking-vars)