Skip to content

best way to save what you love

api: dynamic env reloading from path/url

jjba2d266de79a
diff --git a/api/src/config.js b/api/src/config.js
index e7e29652..a5ebf3d6 100644
--- a/api/src/config.js
+++ b/api/src/config.js
@@ -1,18 +1,34 @@
import { getVersion } from "@imput/version-info";
-import { loadEnvs, validateEnvs } from "./core/env.js";
+import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js";
+import * as cluster from "./misc/cluster.js";
const version = await getVersion();
-let env = loadEnvs();
+const env = loadEnvs();
const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
+export const canonicalEnv = Object.freeze(structuredClone(process.env));
export const setTunnelPort = (port) => env.tunnelPort = port;
export const isCluster = env.instanceCount > 1;
+export const updateEnv = (newEnv) => {
+ // tunnelPort is special and needs to get carried over here
+ newEnv.tunnelPort = env.tunnelPort;
+
+ for (const key in env) {
+ env[key] = newEnv[key];
+ }
+
+ cluster.broadcast({ env_update: newEnv });
+}
await validateEnvs(env);
+if (env.envFile) {
+ setupEnvWatcher();
+}
+
export {
env,
genericUserAgent,
diff --git a/api/src/core/api.js b/api/src/core/api.js
index d8ed8ccd..7c0e15b9 100644
--- a/api/src/core/api.js
+++ b/api/src/core/api.js
@@ -48,19 +48,23 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
const startTime = new Date();
const startTimestamp = startTime.getTime();
- const serverInfo = JSON.stringify({
- cobalt: {
- version: version,
- url: env.apiURL,
- startTime: `${startTimestamp}`,
- durationLimit: env.durationLimit,
- turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
- services: [...env.enabledServices].map(e => {
- return friendlyServiceName(e);
- }),
- },
- git,
- })
+ const getServerInfo = () => {
+ return JSON.stringify({
+ cobalt: {
+ version: version,
+ url: env.apiURL,
+ startTime: `${startTimestamp}`,
+ durationLimit: env.durationLimit,
+ turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
+ services: [...env.enabledServices].map(e => {
+ return friendlyServiceName(e);
+ }),
+ },
+ git,
+ });
+ }
+
+ const serverInfo = getServerInfo();
const handleRateExceeded = (_, res) => {
const { body } = createResponse("error", {
@@ -311,7 +315,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
app.get('/', (_, res) => {
res.type('json');
- res.status(200).send(serverInfo);
+ res.status(200).send(env.envFile ? getServerInfo() : serverInfo);
})
app.get('/favicon.ico', (req, res) => {
@@ -331,10 +335,6 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
if (env.externalProxy) {
- if (env.freebindCIDR) {
- throw new Error('Freebind is not available when external proxy is enabled')
- }
-
setGlobalDispatcher(new ProxyAgent(env.externalProxy))
}
diff --git a/api/src/core/env.js b/api/src/core/env.js
index 34a0d6ef..b50afd1c 100644
--- a/api/src/core/env.js
+++ b/api/src/core/env.js
@@ -1,6 +1,11 @@
import { Constants } from "youtubei.js";
-import { supportsReusePort } from "../misc/cluster.js";
import { services } from "../processing/service-config.js";
+import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
+
+import { FileWatcher } from "../misc/file-watcher.js";
+import { isURL } from "../misc/utils.js";
+import * as cluster from "../misc/cluster.js";
+import { Yellow } from "../misc/console-text.js";
const forceLocalProcessingOptions = ["never", "session", "always"];
@@ -68,6 +73,9 @@ export const loadEnvs = (env = process.env) => {
// "never" | "session" | "always"
forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never",
+
+ envFile: env.API_ENV_FILE,
+ envRemoteReloadInterval: 300,
};
}
@@ -78,7 +86,7 @@ export const validateEnvs = async (env) => {
if (env.instanceCount > 1 && !env.redisURL) {
throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
- } else if (env.instanceCount > 1 && !await supportsReusePort()) {
+ } else if (env.instanceCount > 1 && !await cluster.supportsReusePort()) {
console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
console.error('(or other OS that supports it). for more info, see `reusePort` option on');
@@ -97,4 +105,81 @@ export const validateEnvs = async (env) => {
console.error(`Supported options are are: ${forceLocalProcessingOptions.join(', ')}\n`);
throw new Error("Invalid FORCE_LOCAL_PROCESSING");
}
+
+ if (env.externalProxy && env.freebindCIDR) {
+ throw new Error('freebind is not available when external proxy is enabled')
+ }
+}
+
+const reloadEnvs = async (contents) => {
+ const newEnvs = {};
+
+ for (let line of (await contents).split('\n')) {
+ line = line.trim();
+ if (line === '') {
+ continue;
+ }
+
+ const [ key, value ] = line.split(/=(.+)?/);
+ if (key) {
+ newEnvs[key] = value || '';
+ }
+ }
+
+ const candidate = {
+ ...canonicalEnv,
+ ...newEnvs,
+ };
+
+ const parsed = loadEnvs(candidate);
+ await validateEnvs(parsed);
+ updateEnv(parsed);
+}
+
+const wrapReload = (contents) => {
+ reloadEnvs(contents)
+ .catch((e) => {
+ console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`);
+ console.error('Error:', e);
+ });
+}
+
+let watcher;
+const setupWatcherFromFile = (path) => {
+ const load = () => wrapReload(watcher.read());
+
+ if (isURL(path)) {
+ watcher = FileWatcher.fromFileProtocol(path);
+ } else {
+ watcher = new FileWatcher({ path });
+ }
+
+ watcher.on('file-updated', load);
+ load();
+}
+
+const setupWatcherFromFetch = (url) => {
+ const load = () => wrapReload(fetch(url).then(r => r.text()));
+ setInterval(load, currentEnv.envRemoteReloadInterval);
+ load();
+}
+
+export const setupEnvWatcher = () => {
+ if (cluster.isPrimary) {
+ const envFile = currentEnv.envFile;
+ const isFile = !isURL(envFile)
+ || new URL(envFile).protocol === 'file:';
+
+ if (isFile) {
+ setupWatcherFromFile(envFile);
+ } else {
+ setupWatcherFromFetch(envFile);
+ }
+ } else if (cluster.isWorker) {
+ process.on('message', (message) => {
+ if ('env_update' in message) {
+ updateEnv(message.env_update);
+ }
+ });
+ }
}
diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js
index 62bf6351..1cde3cdc 100644
--- a/api/src/misc/utils.js
+++ b/api/src/misc/utils.js
@@ -52,3 +52,12 @@ export function splitFilenameExtension(filename) {
export function zip(a, b) {
return a.map((value, i) => [ value, b[i] ]);
}
+
+export function isURL(input) {
+ try {
+ new URL(input);
+ return true;
+ } catch {
+ return false;
+ }
+}
diff --git a/docs/api-env-variables.md b/docs/api-env-variables.md
index d44e1cab..92e7a6cf 100644
--- a/docs/api-env-variables.md
+++ b/docs/api-env-variables.md
@@ -13,6 +13,7 @@ this document is not final and will expand over time. feel free to improve it!
| API_REDIS_URL | | `redis://localhost:6379` |
| DISABLED_SERVICES | | `bilibili,youtube` |
| FORCE_LOCAL_PROCESSING | `never` | `always` |
+| API_ENV_FILE | | `/.env` |
[*view details*](#general)
@@ -111,6 +112,9 @@ when set to `session`, only requests from session (Bearer token) clients will be
when set to `always`, all requests will be forced to use on-device processing, no matter the preference.
+### API_ENV_FILE
+the URL or local path to a `key=value`-style environment variable file. this is used for dynamically reloading environment variables. **not all environment variables are able to be updated by this.** (e.g. the ratelimiters are instantiated when starting cobalt, and cannot be changed)
+
## networking
[*jump to the table*](#networking-vars)