Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Improve server configuration (#284)
Hakan Shehu9e7e21d3366c
diff --git a/.gitignore b/.gitignoreindex 2d1171b3..add8286e 100644--- a/.gitignore+++ b/.gitignore@@ -74,11 +74,16 @@ web_modules/# dotenv environment variable files.env-.env.development.local-.env.test.local-.env.production.local+.env.development+.env.test+.env.production.env.local+config.json+config.local.json+config.development.json+config.production.json+# parcel-bundler cache (https://parceljs.org/).cache.parcel-cachediff --git a/apps/server/.env.example b/apps/server/.env.exampleindex 0220f218..a189968b 100644--- a/apps/server/.env.example+++ b/apps/server/.env.example@@ -3,35 +3,15 @@# ------------------------------------------------------------------# Configuration now comes exclusively from config.json. Environment# variables are only read when config.json references them via pointers.-# Use:-# env://VAR_NAME → load from env (append ? to make optional)-# file://path/to/file → load file contents (relative to config.json unless absolute)-# This file is therefore just an example of the required secrets pointed to by config.json.+# Use: env://VAR_NAME → load from env# ------------------------------------------------------------------# ───────────────────────────────────────────────────────────────-# Required secrets referenced by config.json+# Required secrets referenced by default configurations# ───────────────────────────────────────────────────────────────POSTGRES_URL=postgres://colanode_user:postgrespass123@localhost:5432/colanode_dbREDIS_URL=redis://:your_valkey_password@localhost:6379/0-# ───────────────────────────────────────────────────────────────-# Optional: PostgreSQL TLS (used if config.json points to env://POSTGRES_SSL_*)-# ───────────────────────────────────────────────────────────────-# POSTGRES_SSL_CA=-# POSTGRES_SSL_KEY=-# POSTGRES_SSL_CERT=--# ───────────────────────────────────────────────────────────────-# Optional: Google OAuth (account.google.* section)-# ───────────────────────────────────────────────────────────────-# ACCOUNT_GOOGLE_CLIENT_ID=-# ACCOUNT_GOOGLE_CLIENT_SECRET=--# ───────────────────────────────────────────────────────────────-# Optional: Email configuration (email section)-# ───────────────────────────────────────────────────────────────-# EMAIL_FROM=-# EMAIL_SMTP_HOST=-# EMAIL_SMTP_USER=-# EMAIL_SMTP_PASSWORD=+# Create a config.local.json (you can copy see an example in config.example.json)+# and add the following line to use it when running locally+# CONFIG=./config.local.json\ No newline at end of filediff --git a/apps/server/config.example.json b/apps/server/config.example.jsonnew file mode 100644index 00000000..9bc26ee5--- /dev/null+++ b/apps/server/config.example.json@@ -0,0 +1,100 @@+{+ "_comment": "Colanode Server Configuration - Copy this file to config.local.json for local development or use it as a template for your self-hosted server",+ "_docs": "https://colanode.com/docs/self-hosting/configuration/",++ "name": "Colanode Local",+ "mode": "standalone",+ "cors": {+ "origin": "http://localhost:4000",+ "maxAge": 7200+ },+ "account": {+ "verificationType": "automatic",+ "otpTimeout": 600,+ "google": {+ "enabled": false,+ "clientId": "env://ACCOUNT_GOOGLE_CLIENT_ID",+ "clientSecret": "env://ACCOUNT_GOOGLE_CLIENT_SECRET"+ }+ },+ "postgres": {+ "url": "env://POSTGRES_URL",+ "ssl": {+ "ca": "env://POSTGRES_SSL_CA",+ "key": "env://POSTGRES_SSL_KEY",+ "cert": "env://POSTGRES_SSL_CERT"+ }+ },+ "redis": {+ "url": "env://REDIS_URL",+ "db": 0,+ "eventsChannel": "events"+ },+ "storage": {+ "tus": {+ "locker": {+ "type": "redis",+ "prefix": "colanode:tus:lock"+ },+ "cache": {+ "type": "none"+ }+ },+ "provider": {+ "type": "s3",+ "endpoint": "env://S3_ENDPOINT",+ "accessKey": "env://S3_ACCESS_KEY",+ "secretKey": "env://S3_SECRET_KEY",+ "bucket": "env://S3_BUCKET",+ "region": "env://S3_REGION",+ "forcePathStyle": false+ }+ },+ "email": {+ "enabled": false,+ "from": {+ "email": "[email protected]",+ "name": "Colanode"+ },+ "provider": {+ "type": "smtp",+ "host": "env://EMAIL_SMTP_HOST",+ "port": 587,+ "secure": false,+ "auth": {+ "user": "env://EMAIL_SMTP_USER",+ "password": "env://EMAIL_SMTP_PASSWORD"+ }+ }+ },+ "jobs": {+ "queue": {+ "name": "jobs",+ "prefix": "colanode"+ },+ "nodeUpdatesMerge": {+ "enabled": false,+ "cron": "0 5 */2 * * *",+ "batchSize": 500,+ "mergeWindow": 3600,+ "cutoffWindow": 7200+ },+ "documentUpdatesMerge": {+ "enabled": false,+ "cron": "0 5 */2 * * *",+ "batchSize": 500,+ "mergeWindow": 3600,+ "cutoffWindow": 7200+ },+ "cleanup": {+ "enabled": false,+ "cron": "0 5 */2 * * *"+ }+ },+ "workspace": {+ "maxFileSize": "524288000"+ },+ "logging": {+ "level": "info"+ }+}diff --git a/apps/server/config.json b/apps/server/config.jsondeleted file mode 100644index 91a11f66..00000000--- a/apps/server/config.json+++ /dev/null@@ -1,114 +0,0 @@-{- "_comment": "Colanode Server Configuration - Copy this file to config.json and customize as needed",- "_migration_guide": {- "precedence": "config.json (plus env:// / file:// pointers) > schema defaults",- "env_syntax": "Use 'env://VAR_NAME' to reference environment variables. Add '?' suffix for optional variables (e.g., 'env://VAR_NAME?')",- "file_syntax": "Use 'file://relative/or/absolute/path' to inline file contents (e.g., PEM certificates). Add '?' to make it optional",- "backward_compatibility": "Environment variables are only read when referenced via env://"- },-- "server": {- "_comment": "Server branding and networking configuration",- "name": "Colanode Local",- "avatar": null,- "mode": "standalone",- "pathPrefix": null,- "cors": {- "origin": "http://localhost:4000",- "maxAge": 7200- }- },-- "account": {- "_comment": "User account and authentication configuration",- "verificationType": "automatic",- "otpTimeout": 600,- "google": {- "enabled": false,- "_comment_when_enabled": "When enabled=true, you must provide clientId and clientSecret",- "clientId": "env://ACCOUNT_GOOGLE_CLIENT_ID?",- "clientSecret": "env://ACCOUNT_GOOGLE_CLIENT_SECRET?"- }- },-- "postgres": {- "_comment": "PostgreSQL database connection",- "url": "env://POSTGRES_URL",- "ssl": {- "rejectUnauthorized": null,- "ca": "env://POSTGRES_SSL_CA?",- "key": "env://POSTGRES_SSL_KEY?",- "cert": "env://POSTGRES_SSL_CERT?"- }- },-- "redis": {- "_comment": "Redis connection and namespacing",- "url": "env://REDIS_URL",- "db": 0,- "jobs": {- "name": "jobs",- "prefix": "colanode"- },- "tus": {- "lockPrefix": "colanode:tus:lock",- "kvPrefix": "colanode:tus:kv"- },- "eventsChannel": "events"- },-- "storage": {- "_comment": "File storage backend configuration. Supports: 'file' (local filesystem), 's3' (S3-compatible like AWS S3, MinIO, Cloudflare R2), 'gcs' (Google Cloud Storage), 'azure' (Azure Blob Storage). For file storage: set 'directory' to the path where user files will be stored. For other backends, set STORAGE_TYPE env var and provide backend-specific credentials via env:// references",- "type": "file",- "directory": "./colanode"- },-- "email": {- "_comment": "Email sending configuration. Supported providers: smtp",- "enabled": false,- "from": {- "email": "env://EMAIL_FROM?",- "name": "Colanode"- },- "provider": {- "type": "smtp",- "host": "env://EMAIL_SMTP_HOST?",- "port": 587,- "secure": false,- "auth": {- "user": "env://EMAIL_SMTP_USER?",- "password": "env://EMAIL_SMTP_PASSWORD?"- }- }- },-- "jobs": {- "_comment": "Background job configurations",- "nodeUpdatesMerge": {- "enabled": false,- "_comment_when_enabled": "Merges node update events periodically",- "cron": "0 5 */2 * * *",- "batchSize": 500,- "mergeWindow": 3600,- "cutoffWindow": 7200- },- "documentUpdatesMerge": {- "enabled": false,- "_comment_when_enabled": "Merges document update events periodically",- "cron": "0 5 */2 * * *",- "batchSize": 500,- "mergeWindow": 3600,- "cutoffWindow": 7200- },- "cleanup": {- "enabled": false,- "_comment_when_enabled": "Cleans up expired upload chunks",- "cron": "0 5 */2 * * *"- }- },-- "logging": {- "_comment": "Logging level: trace, debug, info, warn, error, fatal, silent",- "level": "info"- }-}diff --git a/apps/server/src/api/client/plugins/cors.ts b/apps/server/src/api/client/plugins/cors.tsindex 55cf0303..b6ad4368 100644--- a/apps/server/src/api/client/plugins/cors.ts+++ b/apps/server/src/api/client/plugins/cors.ts@@ -5,15 +5,11 @@ import fp from 'fastify-plugin';import { config } from '@colanode/server/lib/config';const corsCallback: FastifyPluginCallback = (fastify, _, done) => {- const origin = config.server.cors.origin.includes(',')- ? config.server.cors.origin.split(',').map((o) => o.trim())- : config.server.cors.origin;-fastify.register(cors, {- origin,+ origin: config.cors.origin,methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'],credentials: true,- maxAge: config.server.cors.maxAge,+ maxAge: config.cors.maxAge,});done();diff --git a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.tsindex 46f2af34..b8c146bf 100644--- a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts+++ b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts@@ -86,7 +86,10 @@ export const fileUploadTusRoute: FastifyPluginCallbackZod = (const tusServer = new Server({path: '/tus',datastore: tusStore,- locker: new RedisLocker(redis, config.redis.tus.lockPrefix),+ locker:+ config.storage.tus.locker.type === 'redis'+ ? new RedisLocker(redis, config.storage.tus.locker.prefix)+ : undefined,async onUploadCreate() {const upload = await database.selectFrom('uploads')diff --git a/apps/server/src/api/config.ts b/apps/server/src/api/config.tsindex f15cd518..1f93800a 100644--- a/apps/server/src/api/config.ts+++ b/apps/server/src/api/config.ts@@ -1,6 +1,6 @@import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod';-import { ServerConfig, serverConfigSchema } from '@colanode/core';+import { build, ServerConfig, serverConfigSchema } from '@colanode/core';import { config } from '@colanode/server/lib/config';export const configGetRoute: FastifyPluginCallbackZod = (instance, _, done) => {@@ -14,12 +14,12 @@ export const configGetRoute: FastifyPluginCallbackZod = (instance, _, done) => {},handler: async (request) => {const output: ServerConfig = {- name: config.server.name,- avatar: config.server.avatar ?? '',- version: config.server.version,- sha: config.server.sha,+ name: config.name,+ avatar: config.avatar ?? '',+ version: build.version,+ sha: build.sha,ip: request.client.ip,- pathPrefix: config.server.pathPrefix,+ pathPrefix: config.pathPrefix,account: {google: config.account.google.enabled? {diff --git a/apps/server/src/api/home.ts b/apps/server/src/api/home.tsindex cf1d3944..26e01e4b 100644--- a/apps/server/src/api/home.ts+++ b/apps/server/src/api/home.ts@@ -1,5 +1,6 @@import { FastifyPluginCallback } from 'fastify';+import { build } from '@colanode/core';import { config } from '@colanode/server/lib/config';import { generateUrl } from '@colanode/server/lib/fastify';import { homeTemplate } from '@colanode/server/templates';@@ -12,10 +13,10 @@ export const homeRoute: FastifyPluginCallback = (instance, _, done) => {const configUrl = generateUrl(request, '/config');const template = homeTemplate({- name: config.server.name,+ name: config.name,url: configUrl,- version: config.server.version,- sha: config.server.sha,+ version: build.version,+ sha: build.sha,});reply.type('text/html').send(template);diff --git a/apps/server/src/api/index.ts b/apps/server/src/api/index.tsindex 771d7d8c..6e6b436e 100644--- a/apps/server/src/api/index.ts+++ b/apps/server/src/api/index.ts@@ -6,7 +6,7 @@ import { homeRoute } from '@colanode/server/api/home';import { config } from '@colanode/server/lib/config';export const apiRoutes: FastifyPluginCallback = (instance, _, done) => {- const prefix = config.server.pathPrefix ? `/${config.server.pathPrefix}` : '';+ const prefix = config.pathPrefix ? `/${config.pathPrefix}` : '';instance.register(homeRoute, { prefix });instance.register(configGetRoute, { prefix });diff --git a/apps/server/src/app.ts b/apps/server/src/app.tsindex 58b5c7dc..01ba1c69 100644--- a/apps/server/src/app.ts+++ b/apps/server/src/app.ts@@ -36,7 +36,7 @@ export const initApp = () => {process.exit(1);}- const path = config.server.pathPrefix ? `/${config.server.pathPrefix}` : '';+ const path = config.pathPrefix ? `/${config.pathPrefix}` : '';logger.info(`Server is running at ${address}${path}`);});};diff --git a/apps/server/src/lib/config/account.ts b/apps/server/src/lib/config/account.tsindex 086d92fd..77bacd01 100644--- a/apps/server/src/lib/config/account.ts+++ b/apps/server/src/lib/config/account.ts@@ -1,5 +1,7 @@import { z } from 'zod/v4';+import { resolveConfigReference } from './utils';+export const accountVerificationTypeSchema = z.enum(['automatic','manual',@@ -10,27 +12,38 @@ export type AccountVerificationType = z.infer<typeof accountVerificationTypeSchema>;-export const googleConfigSchema = z.discriminatedUnion('enabled', [- z.object({- enabled: z.literal(true),- clientId: z.string({- error: 'Google client ID is required when Google login is enabled.',+export const googleConfigSchema = z+ .discriminatedUnion('enabled', [+ z.object({+ enabled: z.literal(true),+ clientId: z+ .string({+ error: 'Google client ID is required when Google login is enabled.',+ })+ .transform(resolveConfigReference),+ clientSecret: z+ .string({+ error:+ 'Google client secret is required when Google login is enabled.',+ })+ .transform(resolveConfigReference),}),- clientSecret: z.string({- error: 'Google client secret is required when Google login is enabled.',+ z.object({+ enabled: z.literal(false),}),- }),- z.object({- enabled: z.literal(false),- }),-]);--export const accountConfigSchema = z.object({- verificationType: accountVerificationTypeSchema.default('manual'),- otpTimeout: z.coerce.number().default(600),- google: googleConfigSchema.default({+ ])+ .prefault({enabled: false,- }),-});+ });++export const accountConfigSchema = z+ .object({+ verificationType: accountVerificationTypeSchema+ .transform(resolveConfigReference)+ .default('automatic'),+ otpTimeout: z.coerce.number().default(600),+ google: googleConfigSchema,+ })+ .prefault({});export type AccountConfig = z.infer<typeof accountConfigSchema>;diff --git a/apps/server/src/lib/config/ai.ts b/apps/server/src/lib/config/ai.tsindex 021e7d2a..e13551e9 100644--- a/apps/server/src/lib/config/ai.ts+++ b/apps/server/src/lib/config/ai.ts@@ -41,47 +41,48 @@ export const retrievalConfigSchema = z.object({export type RetrievalConfig = z.infer<typeof retrievalConfigSchema>;-export const aiConfigSchema = z.discriminatedUnion('enabled', [- z.object({- enabled: z.literal(true),- nodeEmbeddingDelay: z.coerce.number().default(5000),- documentEmbeddingDelay: z.coerce.number().default(10000),- providers: z.object({- openai: aiProviderConfigSchema,- google: aiProviderConfigSchema,+export const aiConfigSchema = z+ .discriminatedUnion('enabled', [+ z.object({+ enabled: z.literal(true),+ nodeEmbeddingDelay: z.coerce.number().default(5000),+ documentEmbeddingDelay: z.coerce.number().default(10000),+ providers: z.object({+ openai: aiProviderConfigSchema,+ google: aiProviderConfigSchema,+ }),+ langfuse: z.object({+ enabled: z.boolean().default(false),+ publicKey: z.string().default(''),+ secretKey: z.string().default(''),+ baseUrl: z.string().default('https://cloud.langfuse.com'),+ }),+ models: z.object({+ queryRewrite: aiModelConfigSchema,+ response: aiModelConfigSchema,+ rerank: aiModelConfigSchema,+ summarization: aiModelConfigSchema,+ contextEnhancer: aiModelConfigSchema,+ noContext: aiModelConfigSchema,+ intentRecognition: aiModelConfigSchema,+ databaseFilter: aiModelConfigSchema,+ }),+ embedding: z.object({+ provider: aiProviderSchema.default('openai'),+ modelName: z.string().default('text-embedding-3-large'),+ dimensions: z.coerce.number().default(2000),+ apiKey: z.string().default(''),+ batchSize: z.coerce.number().default(50),+ }),+ chunking: chunkingConfigSchema,+ retrieval: retrievalConfigSchema,}),- langfuse: z.object({- enabled: z.preprocess(- (val) => val === 'true',- z.boolean().default(false)- ),- publicKey: z.string().default(''),- secretKey: z.string().default(''),- baseUrl: z.string().default('https://cloud.langfuse.com'),+ z.object({+ enabled: z.literal(false),}),- models: z.object({- queryRewrite: aiModelConfigSchema,- response: aiModelConfigSchema,- rerank: aiModelConfigSchema,- summarization: aiModelConfigSchema,- contextEnhancer: aiModelConfigSchema,- noContext: aiModelConfigSchema,- intentRecognition: aiModelConfigSchema,- databaseFilter: aiModelConfigSchema,- }),- embedding: z.object({- provider: aiProviderSchema.default('openai'),- modelName: z.string().default('text-embedding-3-large'),- dimensions: z.coerce.number().default(2000),- apiKey: z.string().default(''),- batchSize: z.coerce.number().default(50),- }),- chunking: chunkingConfigSchema,- retrieval: retrievalConfigSchema,- }),- z.object({- enabled: z.literal(false),- }),-]);+ ])+ .prefault({+ enabled: false,+ });export type AiConfig = z.infer<typeof aiConfigSchema>;diff --git a/apps/server/src/lib/config/cors.ts b/apps/server/src/lib/config/cors.tsnew file mode 100644index 00000000..b8427057--- /dev/null+++ b/apps/server/src/lib/config/cors.ts@@ -0,0 +1,18 @@+import { z } from 'zod/v4';++import { resolveConfigReference } from './utils';++const corsOriginSchema = z.union([+ z.string().transform(resolveConfigReference),+ z.array(z.string().transform(resolveConfigReference)),+ z.boolean(),+]);++export const corsSchema = z+ .object({+ origin: corsOriginSchema.default('http://localhost:4000'),+ maxAge: z.coerce.number().default(7200),+ })+ .prefault({});++export type CorsConfig = z.infer<typeof corsSchema>;diff --git a/apps/server/src/lib/config/email.ts b/apps/server/src/lib/config/email.tsindex c287023d..ee9f8d46 100644--- a/apps/server/src/lib/config/email.ts+++ b/apps/server/src/lib/config/email.ts@@ -1,21 +1,27 @@import { z } from 'zod/v4';+import { resolveConfigReference } from './utils';+const smtpProviderConfigSchema = z.object({type: z.literal('smtp'),host: z.string({- error: 'EMAIL_SMTP_HOST is required when SMTP provider is used',+ error: 'Email SMTP host is required when SMTP provider is used',})- .default('smtp'),+ .transform(resolveConfigReference),port: z.coerce.number().default(587),secure: z.boolean().default(false),auth: z.object({- user: z.string({- error: 'EMAIL_SMTP_USER is required when SMTP provider is used',- }),- password: z.string({- error: 'EMAIL_SMTP_PASSWORD is required when SMTP provider is used',- }),+ user: z+ .string({+ error: 'Email SMTP user is required when SMTP provider is used',+ })+ .transform(resolveConfigReference),+ password: z+ .string({+ error: 'Email SMTP password is required when SMTP provider is used',+ })+ .transform(resolveConfigReference),}),});@@ -25,20 +31,31 @@ export const emailProviderConfigSchema = z.discriminatedUnion('type', [export type EmailProviderConfig = z.infer<typeof emailProviderConfigSchema>;-export const emailConfigSchema = z.discriminatedUnion('enabled', [- z.object({- enabled: z.literal(true),- from: z.object({- email: z.string({- error: 'EMAIL_FROM is required when email is enabled',+export const emailConfigSchema = z+ .discriminatedUnion('enabled', [+ z.object({+ enabled: z.literal(true),+ from: z.object({+ email: z+ .string({+ error: 'Email from email is required when email is enabled',+ })+ .transform(resolveConfigReference),+ name: z+ .string({+ error: 'Email from name is required when email is enabled',+ })+ .default('Colanode')+ .transform(resolveConfigReference),}),- name: z.string().default('Colanode'),+ provider: emailProviderConfigSchema,}),- provider: emailProviderConfigSchema,- }),- z.object({- enabled: z.literal(false),- }),-]);+ z.object({+ enabled: z.literal(false),+ }),+ ])+ .prefault({+ enabled: false,+ });export type EmailConfig = z.infer<typeof emailConfigSchema>;diff --git a/apps/server/src/lib/config/index.ts b/apps/server/src/lib/config/index.tsindex da0d09ad..96634f6e 100644--- a/apps/server/src/lib/config/index.ts+++ b/apps/server/src/lib/config/index.ts@@ -1,19 +1,32 @@+import fs from 'fs';+import { z } from 'zod/v4';import { accountConfigSchema } from './account';import { aiConfigSchema } from './ai';+import { corsSchema } from './cors';import { emailConfigSchema } from './email';import { jobsConfigSchema } from './jobs';-import { loadRawConfig } from './loader';import { loggingConfigSchema } from './logging';import { postgresConfigSchema } from './postgres';import { redisConfigSchema } from './redis';-import { serverConfigSchema } from './server';import { storageConfigSchema } from './storage';+import {+ resolveConfigReference,+ resolveOptionalConfigReference,+} from './utils';import { workspaceConfigSchema } from './workspace';+const serverModeSchema = z.enum(['standalone', 'cluster']);+const configSchema = z.object({- server: serverConfigSchema,+ name: z.string().default('Colanode Server').transform(resolveConfigReference),+ avatar: z.string().optional().transform(resolveOptionalConfigReference),+ mode: serverModeSchema+ .default('standalone')+ .transform(resolveConfigReference),+ pathPrefix: z.string().optional().transform(resolveOptionalConfigReference),+ cors: corsSchema,account: accountConfigSchema,postgres: postgresConfigSchema,redis: redisConfigSchema,@@ -27,9 +40,20 @@ const configSchema = z.object({export type Configuration = z.infer<typeof configSchema>;-const readConfigVariables = (): Configuration => {+const loadConfigFromPath = (configPath: string): Partial<Configuration> => {+ if (!fs.existsSync(configPath)) {+ throw new Error(`Configuration file not found: ${configPath}`);+ }++ const raw = fs.readFileSync(configPath, 'utf-8');+ const parsed = JSON.parse(raw);+ return parsed;+};++const buildConfig = (): Configuration => {try {- const input = loadRawConfig();+ const configPath = process.env.CONFIG;+ const input = configPath ? loadConfigFromPath(configPath) : {};return configSchema.parse(input);} catch (error) {if (error instanceof z.ZodError) {@@ -45,4 +69,4 @@ const readConfigVariables = (): Configuration => {}};-export const config = readConfigVariables();+export const config = buildConfig();diff --git a/apps/server/src/lib/config/jobs.ts b/apps/server/src/lib/config/jobs.tsindex fffb9553..487a7392 100644--- a/apps/server/src/lib/config/jobs.ts+++ b/apps/server/src/lib/config/jobs.ts@@ -1,30 +1,41 @@import ms from 'ms';import { z } from 'zod/v4';+import { resolveConfigReference } from './utils';+const DEFAULT_BATCH_SIZE = 500;const DEFAULT_MERGE_WINDOW = ms('1 hour') / 1000; // in secondsconst DEFAULT_CUTOFF_WINDOW = ms('2 hours') / 1000; // in secondsconst DEFAULT_CRON_PATTERN = '0 5 */2 * * *'; // every 2 hours at the 5th minute-export const nodeUpdatesMergeJobConfigSchema = z.discriminatedUnion('enabled', [- z.object({- enabled: z.literal(true),- cron: z.string().default(DEFAULT_CRON_PATTERN),- batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE),- mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW),- cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW),- }),- z.object({- enabled: z.literal(false),- }),-]);--export const documentUpdatesMergeJobConfigSchema = z.discriminatedUnion(- 'enabled',- [+export const nodeUpdatesMergeJobConfigSchema = z+ .discriminatedUnion('enabled', [+ z.object({+ enabled: z.literal(true),+ cron: z+ .string()+ .default(DEFAULT_CRON_PATTERN)+ .transform(resolveConfigReference),+ batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE),+ mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW),+ cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW),+ }),+ z.object({+ enabled: z.literal(false),+ }),+ ])+ .prefault({+ enabled: false,+ });++export const documentUpdatesMergeJobConfigSchema = z+ .discriminatedUnion('enabled', [z.object({enabled: z.literal(true),- cron: z.string().default(DEFAULT_CRON_PATTERN),+ cron: z+ .string()+ .default(DEFAULT_CRON_PATTERN)+ .transform(resolveConfigReference),batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE),mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW),cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW),@@ -32,23 +43,42 @@ export const documentUpdatesMergeJobConfigSchema = z.discriminatedUnion(z.object({enabled: z.literal(false),}),- ]-);--export const cleanupJobConfigSchema = z.discriminatedUnion('enabled', [- z.object({- enabled: z.literal(true),- cron: z.string().default(DEFAULT_CRON_PATTERN),- }),- z.object({- enabled: z.literal(false),- }),-]);--export const jobsConfigSchema = z.object({- nodeUpdatesMerge: nodeUpdatesMergeJobConfigSchema,- documentUpdatesMerge: documentUpdatesMergeJobConfigSchema,- cleanup: cleanupJobConfigSchema,-});+ ])+ .prefault({+ enabled: false,+ });++export const cleanupJobConfigSchema = z+ .discriminatedUnion('enabled', [+ z.object({+ enabled: z.literal(true),+ cron: z+ .string()+ .default(DEFAULT_CRON_PATTERN)+ .transform(resolveConfigReference),+ }),+ z.object({+ enabled: z.literal(false),+ }),+ ])+ .prefault({+ enabled: false,+ });++export const jobsQueueSchema = z+ .object({+ name: z.string().default('jobs').transform(resolveConfigReference),+ prefix: z.string().default('colanode').transform(resolveConfigReference),+ })+ .prefault({});++export const jobsConfigSchema = z+ .object({+ queue: jobsQueueSchema,+ nodeUpdatesMerge: nodeUpdatesMergeJobConfigSchema,+ documentUpdatesMerge: documentUpdatesMergeJobConfigSchema,+ cleanup: cleanupJobConfigSchema,+ })+ .prefault({});export type JobsConfig = z.infer<typeof jobsConfigSchema>;diff --git a/apps/server/src/lib/config/loader.ts b/apps/server/src/lib/config/loader.tsdeleted file mode 100644index 471bb11b..00000000--- a/apps/server/src/lib/config/loader.ts+++ /dev/null@@ -1,177 +0,0 @@-import fs from 'fs';-import { fileURLToPath } from 'node:url';-import path from 'path';--import { Configuration } from './index';--type ConfigSource = Partial<Configuration>;--const ENV_POINTER_PATTERN = /^env:\/\/([A-Z0-9_]+)(\?)?$/;-const FILE_POINTER_PATTERN = /^file:\/\/(.+?)(\?)?$/;--export class MissingEnvVarError extends Error {- constructor(varName: string) {- super(`Missing required environment variable: ${varName}`);- this.name = 'MissingEnvVarError';- }-}--export class MissingFileError extends Error {- constructor(filePath: string) {- super(`Missing required configuration file: ${filePath}`);- this.name = 'MissingFileError';- }-}--const isRecord = (value: unknown): value is Record<string, unknown> => {- return typeof value === 'object' && value !== null && !Array.isArray(value);-};--type NormalizeContext = {- configDir: string;-};--const normalizeValue = (value: unknown, ctx: NormalizeContext): unknown => {- if (value === null) {- return undefined;- }-- if (typeof value === 'string') {- return resolvePointerValue(value, ctx);- }-- if (Array.isArray(value)) {- return value.map((item) => normalizeValue(item, ctx));- }-- if (isRecord(value)) {- const normalized: Record<string, unknown> = {};-- for (const [key, nested] of Object.entries(value)) {- // Skip fields starting with underscore (documentation fields)- if (key.startsWith('_')) {- continue;- }-- const processed = normalizeValue(nested, ctx);-- if (processed !== undefined) {- normalized[key] = processed;- }- }-- return normalized;- }-- return value;-};--const candidateConfigDirectories = (): string[] => {- const cwd = process.cwd();- const moduleDir = path.dirname(fileURLToPath(import.meta.url));- const serverRoot = path.resolve(moduleDir, '../../..');- const candidates = [cwd, path.join(cwd, 'apps/server'), serverRoot];-- return Array.from(new Set(candidates.map((dir) => path.resolve(dir))));-};--const findConfigFile = (filename: string): string | undefined => {- for (const dir of candidateConfigDirectories()) {- const candidate = path.join(dir, filename);-- if (fs.existsSync(candidate)) {- return candidate;- }- }-- return undefined;-};--const readJsonFile = (filePath: string): Record<string, unknown> => {- try {- const raw = fs.readFileSync(filePath, 'utf-8');- const parsed = JSON.parse(raw);-- if (!isRecord(parsed)) {- throw new Error('configuration file must contain a JSON object');- }-- return parsed;- } catch (error) {- const reason = error instanceof Error ? error.message : String(error);- throw new Error(`Failed to read ${filePath}: ${reason}`);- }-};--const resolvePointerValue = (value: string, ctx: NormalizeContext): unknown => {- const envMatch = value.match(ENV_POINTER_PATTERN);-- if (envMatch) {- const [, envName, optionalFlag] = envMatch;- const envValue = process.env[envName!];- const optional = optionalFlag === '?';-- if (envValue === undefined) {- if (optional) {- return undefined;- }-- throw new MissingEnvVarError(envName!);- }-- return envValue;- }-- const fileMatch = value.match(FILE_POINTER_PATTERN);-- if (fileMatch) {- const [, rawPath, optionalFlag] = fileMatch;- const optional = optionalFlag === '?';- const resolvedPath = path.isAbsolute(rawPath!)- ? rawPath!- : path.resolve(ctx.configDir, rawPath!);-- if (!fs.existsSync(resolvedPath)) {- if (optional) {- return undefined;- }-- throw new MissingFileError(resolvedPath);- }-- try {- return fs.readFileSync(resolvedPath, 'utf-8');- } catch (error) {- const reason = error instanceof Error ? error.message : String(error);- throw new Error(`Failed to read ${resolvedPath}: ${reason}`);- }- }-- return value;-};--export const resolveEnvPointers = (- value: unknown,- ctx: NormalizeContext-): unknown => {- return normalizeValue(value, ctx);-};--export const loadRawConfig = (): ConfigSource => {- const configPath = findConfigFile('config.json');-- if (!configPath) {- throw new Error(- [- 'Unable to find config.json.',- 'Copy apps/server/config.json (or mount your own) so the server has a configuration file.',- ].join(' ')- );- }-- const jsonConfig = readJsonFile(configPath) as ConfigSource;-- return normalizeValue(jsonConfig, {- configDir: path.dirname(configPath),- }) as ConfigSource;-};diff --git a/apps/server/src/lib/config/logging.ts b/apps/server/src/lib/config/logging.tsindex 8a84b14e..810f1c02 100644--- a/apps/server/src/lib/config/logging.ts+++ b/apps/server/src/lib/config/logging.ts@@ -1,9 +1,14 @@import { z } from 'zod/v4';-export const loggingConfigSchema = z.object({- level: z- .enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'silent'])- .default('info'),-});+import { resolveConfigReference } from './utils';++export const loggingConfigSchema = z+ .object({+ level: z+ .enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'silent'])+ .default('info')+ .transform(resolveConfigReference),+ })+ .prefault({});export type LoggingConfig = z.infer<typeof loggingConfigSchema>;diff --git a/apps/server/src/lib/config/postgres.ts b/apps/server/src/lib/config/postgres.tsindex e38288f7..f92b9a01 100644--- a/apps/server/src/lib/config/postgres.ts+++ b/apps/server/src/lib/config/postgres.ts@@ -1,19 +1,28 @@import { z } from 'zod/v4';-export const postgresConfigSchema = z.object({- url: z.string({- error:- 'POSTGRES_URL is required (e.g. postgres://postgres:postgres@localhost:5432/postgres)',- }),- ssl: z.object({- rejectUnauthorized: z.preprocess(- (val) => (val === undefined ? undefined : val === 'true'),- z.boolean().optional()- ),- ca: z.string().optional(),- key: z.string().optional(),- cert: z.string().optional(),- }),-});+import {+ resolveConfigReference,+ resolveOptionalConfigReference,+} from './utils';++export const postgresConfigSchema = z+ .object({+ url: z+ .string({+ error:+ 'Postgres URL is required (e.g. postgres://postgres:postgres@localhost:5432/postgres)',+ })+ .default('env://POSTGRES_URL')+ .transform(resolveConfigReference),+ ssl: z+ .object({+ rejectUnauthorized: z.boolean().optional(),+ ca: z.string().optional().transform(resolveOptionalConfigReference),+ key: z.string().optional().transform(resolveOptionalConfigReference),+ cert: z.string().optional().transform(resolveOptionalConfigReference),+ })+ .optional(),+ })+ .prefault({});export type PostgresConfig = z.infer<typeof postgresConfigSchema>;diff --git a/apps/server/src/lib/config/redis.ts b/apps/server/src/lib/config/redis.tsindex a820d62f..b7949eef 100644--- a/apps/server/src/lib/config/redis.ts+++ b/apps/server/src/lib/config/redis.ts@@ -1,17 +1,16 @@import { z } from 'zod/v4';-export const redisConfigSchema = z.object({- url: z.string({ error: 'REDIS_URL is required' }),- db: z.coerce.number().default(0),- jobs: z.object({- name: z.string().optional().default('jobs'),- prefix: z.string().optional().default('colanode'),- }),- tus: z.object({- lockPrefix: z.string().optional().default('colanode:tus:lock'),- kvPrefix: z.string().optional().default('colanode:tus:kv'),- }),- eventsChannel: z.string().optional().default('events'),-});+import { resolveConfigReference } from './utils';++export const redisConfigSchema = z+ .object({+ url: z+ .string({ error: 'Redis URL is required' })+ .default('env://REDIS_URL')+ .transform(resolveConfigReference),+ db: z.coerce.number().default(0),+ eventsChannel: z.string().optional().default('events'),+ })+ .prefault({});export type RedisConfig = z.infer<typeof redisConfigSchema>;diff --git a/apps/server/src/lib/config/server.ts b/apps/server/src/lib/config/server.tsdeleted file mode 100644index 25b8b0de..00000000--- a/apps/server/src/lib/config/server.ts+++ /dev/null@@ -1,21 +0,0 @@-import { z } from 'zod/v4';--import { build } from '@colanode/core';--const serverModeSchema = z.enum(['standalone', 'cluster']);-export type ServerMode = z.infer<typeof serverModeSchema>;--export const serverConfigSchema = z.object({- version: z.string().default(build.version),- sha: z.string().default(build.sha),- name: z.string().default('Colanode Server'),- avatar: z.string().optional(),- mode: serverModeSchema.default('standalone'),- pathPrefix: z.string().optional(),- cors: z.object({- origin: z.string().default('http://localhost:4000'),- maxAge: z.coerce.number().default(7200),- }),-});--export type ServerConfig = z.infer<typeof serverConfigSchema>;diff --git a/apps/server/src/lib/config/storage.ts b/apps/server/src/lib/config/storage.tsindex 68ffb274..ed073a27 100644--- a/apps/server/src/lib/config/storage.ts+++ b/apps/server/src/lib/config/storage.ts@@ -1,45 +1,131 @@import { z } from 'zod/v4';-const s3StorageConfigSchema = z.object({+import { resolveConfigReference } from './utils';++const s3StorageProviderConfigSchema = z.object({type: z.literal('s3'),- endpoint: z.string({ error: 'STORAGE_S3_ENDPOINT is required' }),- accessKey: z.string({ error: 'STORAGE_S3_ACCESS_KEY is required' }),- secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }),- bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }),- region: z.string({ error: 'STORAGE_S3_REGION is required' }),+ endpoint: z+ .string({ error: 'Storage S3 endpoint is required' })+ .transform(resolveConfigReference),+ accessKey: z+ .string({ error: 'Storage S3 access key is required' })+ .transform(resolveConfigReference),+ secretKey: z+ .string({ error: 'Storage S3 secret key is required' })+ .transform(resolveConfigReference),+ bucket: z+ .string({ error: 'Storage S3 bucket is required' })+ .transform(resolveConfigReference),+ region: z.string({ error: 'Storage S3 region is required' }),forcePathStyle: z.boolean().optional(),});-const fileStorageConfigSchema = z.object({+const fileStorageProviderConfigSchema = z.object({type: z.literal('file'),- directory: z.string({ error: 'STORAGE_FILE_DIRECTORY is required' }),+ directory: z+ .string({ error: 'Storage file directory is required' })+ .default('./colanode')+ .transform(resolveConfigReference),});-const gcsStorageConfigSchema = z.object({+const gcsStorageProviderConfigSchema = z.object({type: z.literal('gcs'),- bucket: z.string({ error: 'STORAGE_GCS_BUCKET is required' }),- projectId: z.string({ error: 'STORAGE_GCS_PROJECT_ID is required' }),- credentials: z.string({ error: 'STORAGE_GCS_CREDENTIALS is required' }),+ bucket: z+ .string({ error: 'Storage GCS bucket is required' })+ .transform(resolveConfigReference),+ projectId: z+ .string({ error: 'Storage GCS project ID is required' })+ .transform(resolveConfigReference),+ credentials: z+ .string({ error: 'Storage GCS credentials is required' })+ .transform(resolveConfigReference),});-const azureStorageConfigSchema = z.object({+const azureStorageProviderConfigSchema = z.object({type: z.literal('azure'),- account: z.string({ error: 'STORAGE_AZURE_ACCOUNT is required' }),- accountKey: z.string({ error: 'STORAGE_AZURE_ACCOUNT_KEY is required' }),- containerName: z.string({- error: 'STORAGE_AZURE_CONTAINER_NAME is required',- }),+ account: z+ .string({ error: 'Storage Azure account is required' })+ .transform(resolveConfigReference),+ accountKey: z+ .string({ error: 'Storage Azure account key is required' })+ .transform(resolveConfigReference),+ containerName: z+ .string({ error: 'Storage Azure container name is required' })+ .transform(resolveConfigReference),});-export const storageConfigSchema = z.discriminatedUnion('type', [- s3StorageConfigSchema,- fileStorageConfigSchema,- gcsStorageConfigSchema,- azureStorageConfigSchema,-]);+export const storageProviderConfigSchema = z+ .discriminatedUnion('type', [+ s3StorageProviderConfigSchema,+ fileStorageProviderConfigSchema,+ gcsStorageProviderConfigSchema,+ azureStorageProviderConfigSchema,+ ])+ .prefault({+ type: 'file',+ });++export const tusLockerSchema = z+ .discriminatedUnion('type', [+ z.object({+ type: z.literal('redis'),+ prefix: z+ .string()+ .default('colanode:tus:lock')+ .transform(resolveConfigReference),+ }),+ z.object({+ type: z.literal('memory'),+ }),+ ])+ .prefault({+ type: 'memory',+ });++export const tusCacheSchema = z+ .discriminatedUnion('type', [+ z.object({ type: z.literal('none') }),+ z.object({+ type: z.literal('redis'),+ prefix: z+ .string()+ .default('colanode:tus:kv')+ .transform(resolveConfigReference),+ }),+ ])+ .prefault({+ type: 'none',+ });++export const tusConfigSchema = z+ .object({+ locker: tusLockerSchema,+ cache: tusCacheSchema,+ })+ .prefault({});++export const storageConfigSchema = z+ .object({+ tus: tusConfigSchema,+ provider: storageProviderConfigSchema,+ })+ .prefault({});++export type TusLockerConfig = z.infer<typeof tusLockerSchema>;+export type TusCacheConfig = z.infer<typeof tusCacheSchema>;+export type TusConfig = z.infer<typeof tusConfigSchema>;+export type StorageProviderConfig = z.infer<typeof storageProviderConfigSchema>;+export type S3StorageProviderConfig = z.infer<+ typeof s3StorageProviderConfigSchema+>;+export type FileStorageProviderConfig = z.infer<+ typeof fileStorageProviderConfigSchema+>;+export type GCSStorageProviderConfig = z.infer<+ typeof gcsStorageProviderConfigSchema+>;+export type AzureStorageProviderConfig = z.infer<+ typeof azureStorageProviderConfigSchema+>;export type StorageConfig = z.infer<typeof storageConfigSchema>;-export type S3StorageConfig = z.infer<typeof s3StorageConfigSchema>;-export type FileStorageConfig = z.infer<typeof fileStorageConfigSchema>;-export type GCSStorageConfig = z.infer<typeof gcsStorageConfigSchema>;-export type AzureStorageConfig = z.infer<typeof azureStorageConfigSchema>;diff --git a/apps/server/src/lib/config/utils.ts b/apps/server/src/lib/config/utils.tsnew file mode 100644index 00000000..682ab071--- /dev/null+++ b/apps/server/src/lib/config/utils.ts@@ -0,0 +1,58 @@+import fs from 'node:fs';++import { z } from 'zod/v4';++const resolveValue = (+ val: string,+ ctx: z.core.$RefinementCtx<string>+): string => {+ if (val.startsWith('env://')) {+ const envName = val.slice(6);+ const envValue = process.env[envName];+ if (envValue === undefined) {+ ctx.addIssue({+ code: 'invalid_value',+ message: `Environment variable "${envName}" is not set`,+ values: [envName],+ });+ return z.NEVER;+ }++ return envValue;+ }++ if (val.startsWith('file://')) {+ const filePath = val.slice(7);+ try {+ return fs.readFileSync(filePath, 'utf-8').trim();+ } catch (err: unknown) {+ const error = err instanceof Error ? err : new Error(String(err));+ ctx.addIssue({+ code: 'invalid_value',+ message: `Failed to read secret from file "${filePath}": ${error.message}`,+ values: [filePath],+ });+ return z.NEVER;+ }+ }++ return val;+};++export const resolveConfigReference = (+ val: string,+ ctx: z.core.$RefinementCtx<string>+): string => {+ return resolveValue(val, ctx);+};++export const resolveOptionalConfigReference = (+ val: string | undefined,+ ctx: z.core.$RefinementCtx<string | undefined>+): string | undefined => {+ if (val === undefined) {+ return undefined;+ }++ return resolveValue(val, ctx as z.core.$RefinementCtx<string>);+};diff --git a/apps/server/src/lib/config/workspace.ts b/apps/server/src/lib/config/workspace.tsindex e425d462..1b2bcf94 100644--- a/apps/server/src/lib/config/workspace.ts+++ b/apps/server/src/lib/config/workspace.ts@@ -1,7 +1,9 @@import { z } from 'zod/v4';-export const workspaceConfigSchema = z.object({- maxFileSize: z.string().optional().nullable(),-});+export const workspaceConfigSchema = z+ .object({+ maxFileSize: z.string().optional(),+ })+ .prefault({});export type WorkspaceConfig = z.infer<typeof workspaceConfigSchema>;diff --git a/apps/server/src/lib/event-bus.ts b/apps/server/src/lib/event-bus.tsindex f10bf2c7..459a2d7f 100644--- a/apps/server/src/lib/event-bus.ts+++ b/apps/server/src/lib/event-bus.ts@@ -37,7 +37,7 @@ export class EventBusService {this.initialized = true;- if (config.server.mode === 'standalone') {+ if (config.mode === 'standalone') {return;}@@ -72,7 +72,7 @@ export class EventBusService {public publish(event: Event) {this.processEvent(event);- if (config.server.mode === 'standalone') {+ if (config.mode === 'standalone') {return;}diff --git a/apps/server/src/lib/fastify.ts b/apps/server/src/lib/fastify.tsindex 822265e4..16c05020 100644--- a/apps/server/src/lib/fastify.ts+++ b/apps/server/src/lib/fastify.ts@@ -8,7 +8,7 @@ export const generateUrl = (request: FastifyRequest, path: string) => {? `:${request.port}`: '';- const prefix = config.server.pathPrefix ? `/${config.server.pathPrefix}` : '';+ const prefix = config.pathPrefix ? `/${config.pathPrefix}` : '';return `${request.protocol}://${request.hostname}${port}${prefix}${path}`;};diff --git a/apps/server/src/lib/storage/azure.ts b/apps/server/src/lib/storage/azure.tsindex b2479328..1eafe4d5 100644--- a/apps/server/src/lib/storage/azure.ts+++ b/apps/server/src/lib/storage/azure.ts@@ -8,38 +8,45 @@ import {import { AzureStore } from '@tus/azure-store';import { DataStore } from '@tus/server';-import type { AzureStorageConfig } from '@colanode/server/lib/config/storage';+import { redis } from '@colanode/server/data/redis';+import type {+ AzureStorageProviderConfig,+ TusConfig,+} from '@colanode/server/lib/config/storage';+import { RedisKvStore } from '@colanode/server/lib/storage/tus/redis-kv';import type { Storage } from './core';export class AzureBlobStorage implements Storage {private readonly containerName: string;private readonly blobServiceClient: BlobServiceClient;- private readonly config: AzureStorageConfig;- private readonly azureStore: AzureStore;+ private readonly store: AzureStore;- constructor(config: AzureStorageConfig) {- this.config = { ...config };+ constructor(config: AzureStorageProviderConfig, tusConfig: TusConfig) {const sharedKeyCredential = new StorageSharedKeyCredential(- this.config.account,- this.config.accountKey+ config.account,+ config.accountKey);this.blobServiceClient = new BlobServiceClient(- `https://${this.config.account}.blob.core.windows.net`,+ `https://${config.account}.blob.core.windows.net`,sharedKeyCredential);- this.containerName = this.config.containerName;+ this.containerName = config.containerName;- this.azureStore = new AzureStore({- account: this.config.account,- accountKey: this.config.accountKey,+ this.store = new AzureStore({+ account: config.account,+ accountKey: config.accountKey,containerName: this.containerName,+ cache:+ tusConfig.cache.type === 'redis'+ ? new RedisKvStore(redis, tusConfig.cache.prefix)+ : undefined,});}public get tusStore(): DataStore {- return this.azureStore;+ return this.store;}private getBlockBlobClient(path: string): BlockBlobClient {diff --git a/apps/server/src/lib/storage/fs.ts b/apps/server/src/lib/storage/fs.tsindex e3de2a45..8caf0e7a 100644--- a/apps/server/src/lib/storage/fs.ts+++ b/apps/server/src/lib/storage/fs.ts@@ -4,17 +4,32 @@ import { Readable } from 'stream';import { FileStore } from '@tus/file-store';import { DataStore } from '@tus/server';-import type { FileStorageConfig } from '@colanode/server/lib/config/storage';+import { redis } from '@colanode/server/data/redis';+import type {+ FileStorageProviderConfig,+ TusConfig,+} from '@colanode/server/lib/config/storage';+import { RedisKvStore } from '@colanode/server/lib/storage/tus/redis-kv';import type { Storage } from './core';export class FileSystemStorage implements Storage {private readonly directory: string;- public readonly tusStore: DataStore;+ private readonly store: DataStore;- constructor(config: FileStorageConfig) {+ constructor(config: FileStorageProviderConfig, tusConfig: TusConfig) {this.directory = config.directory;- this.tusStore = new FileStore({ directory: this.directory });+ this.store = new FileStore({+ directory: this.directory,+ configstore:+ tusConfig.cache.type === 'redis'+ ? new RedisKvStore(redis, tusConfig.cache.prefix)+ : undefined,+ });+ }++ public get tusStore(): DataStore {+ return this.store;}async download(diff --git a/apps/server/src/lib/storage/gcs.ts b/apps/server/src/lib/storage/gcs.tsindex f900ae80..c2e46ac6 100644--- a/apps/server/src/lib/storage/gcs.ts+++ b/apps/server/src/lib/storage/gcs.ts@@ -4,7 +4,7 @@ import { Storage as GoogleStorage, Bucket, File } from '@google-cloud/storage';import { GCSStore } from '@tus/gcs-store';import { DataStore } from '@tus/server';-import type { GCSStorageConfig } from '@colanode/server/lib/config/storage';+import type { GCSStorageProviderConfig } from '@colanode/server/lib/config/storage';import type { Storage } from './core';@@ -12,7 +12,7 @@ export class GCSStorage implements Storage {private readonly bucket: Bucket;private readonly gcsStore: GCSStore;- constructor(config: GCSStorageConfig) {+ constructor(config: GCSStorageProviderConfig) {const storage = new GoogleStorage({projectId: config.projectId,keyFilename: config.credentials,diff --git a/apps/server/src/lib/storage/index.ts b/apps/server/src/lib/storage/index.tsindex 6d921c4b..18e400a6 100644--- a/apps/server/src/lib/storage/index.ts+++ b/apps/server/src/lib/storage/index.ts@@ -7,19 +7,19 @@ import { FileSystemStorage } from './fs';import { GCSStorage } from './gcs';import { S3Storage } from './s3';-const buildStorage = (storageConfig: StorageConfig): Storage => {- switch (storageConfig.type) {+const buildStorage = (config: StorageConfig): Storage => {+ switch (config.provider.type) {case 'file':- return new FileSystemStorage(storageConfig);+ return new FileSystemStorage(config.provider, config.tus);case 's3':- return new S3Storage(storageConfig);+ return new S3Storage(config.provider, config.tus);case 'gcs':- return new GCSStorage(storageConfig);+ return new GCSStorage(config.provider);case 'azure':- return new AzureBlobStorage(storageConfig);+ return new AzureBlobStorage(config.provider, config.tus);default:throw new Error(- `Unsupported storage type: ${JSON.stringify(storageConfig)}`+ `Unsupported storage provider: ${JSON.stringify(config.provider)}`);}};diff --git a/apps/server/src/lib/storage/s3.ts b/apps/server/src/lib/storage/s3.tsindex c69ae6c2..0fb33afd 100644--- a/apps/server/src/lib/storage/s3.ts+++ b/apps/server/src/lib/storage/s3.ts@@ -11,8 +11,10 @@ import { DataStore } from '@tus/server';import { FILE_UPLOAD_PART_SIZE } from '@colanode/core';import { redis } from '@colanode/server/data/redis';-import { config } from '@colanode/server/lib/config';-import type { S3StorageConfig } from '@colanode/server/lib/config/storage';+import type {+ S3StorageProviderConfig,+ TusConfig,+} from '@colanode/server/lib/config/storage';import { RedisKvStore } from '@colanode/server/lib/storage/tus/redis-kv';import type { Storage } from './core';@@ -20,43 +22,44 @@ import type { Storage } from './core';export class S3Storage implements Storage {private readonly client: S3Client;private readonly bucket: string;- private readonly s3Config: S3StorageConfig;- private readonly s3Store: S3Store;- private readonly redisKv: RedisKvStore<MetadataValue>;+ private readonly store: DataStore;+ private readonly cache?: RedisKvStore<MetadataValue>;- constructor(s3Config: S3StorageConfig) {- this.s3Config = { ...s3Config };+ constructor(config: S3StorageProviderConfig, tusConfig: TusConfig) {this.client = new S3Client({- endpoint: this.s3Config.endpoint,- region: this.s3Config.region,+ endpoint: config.endpoint,+ region: config.region,credentials: {- accessKeyId: this.s3Config.accessKey,- secretAccessKey: this.s3Config.secretKey,+ accessKeyId: config.accessKey,+ secretAccessKey: config.secretKey,},- forcePathStyle: this.s3Config.forcePathStyle,+ forcePathStyle: config.forcePathStyle,});- this.bucket = this.s3Config.bucket;+ this.bucket = config.bucket;- this.redisKv = new RedisKvStore(redis, config.redis.tus.kvPrefix);- this.s3Store = new S3Store({+ if (tusConfig.cache.type === 'redis') {+ this.cache = new RedisKvStore(redis, tusConfig.cache.prefix);+ }++ this.store = new S3Store({partSize: FILE_UPLOAD_PART_SIZE,- cache: this.redisKv,+ cache: this.cache,s3ClientConfig: {bucket: this.bucket,- endpoint: this.s3Config.endpoint,- region: this.s3Config.region,- forcePathStyle: this.s3Config.forcePathStyle,+ endpoint: config.endpoint,+ region: config.region,+ forcePathStyle: config.forcePathStyle,credentials: {- accessKeyId: this.s3Config.accessKey,- secretAccessKey: this.s3Config.secretKey,+ accessKeyId: config.accessKey,+ secretAccessKey: config.secretKey,},},});}public get tusStore(): DataStore {- return this.s3Store;+ return this.store;}public async download(@@ -78,10 +81,13 @@ export class S3Storage implements Storage {public async delete(path: string): Promise<void> {const command = new DeleteObjectCommand({ Bucket: this.bucket, Key: path });await this.client.send(command);- await this.redisKv.delete(path);- const infoPath = `${path}.info`;- await this.redisKv.delete(infoPath);+ if (this.cache) {+ await this.cache.delete(path);++ const infoPath = `${path}.info`;+ await this.cache.delete(infoPath);+ }}public async upload(diff --git a/apps/server/src/services/job-service.ts b/apps/server/src/services/job-service.tsindex 4dfc6dce..6ca316b0 100644--- a/apps/server/src/services/job-service.ts+++ b/apps/server/src/services/job-service.ts@@ -16,15 +16,15 @@ class JobService {// for more information, see: https://docs.bullmq.io/bull/patterns/redis-cluster- private readonly queueName = config.redis.jobs.name;- private readonly prefix = `{${config.redis.jobs.prefix}}`;+ private readonly queueName = config.jobs.queue.name;+ private readonly prefix = `{${config.jobs.queue.prefix}}`;public async initQueue(): Promise<void> {if (this.jobQueue) {return;}- this.jobQueue = new Queue(this.queueName, {+ this.jobQueue = new Queue(this.queueName!, {prefix: this.prefix,connection: {db: config.redis.db,diff --git a/hosting/docker/docker-compose.yaml b/hosting/docker/docker-compose.yamlindex 6d6fde56..2364ad55 100644--- a/hosting/docker/docker-compose.yaml+++ b/hosting/docker/docker-compose.yaml@@ -95,23 +95,20 @@ services:# ------------------------------------------------------------------NODE_ENV: production- # Required env:// secrets (config.json marks these as env://VAR)+ # Required env:// secrets (default configuration references these as env://VAR)POSTGRES_URL: postgres://colanode_user:postgrespass123@postgres:5432/colanode_dbREDIS_URL: redis://:your_valkey_password@valkey:6379/0- # Optional env://? secrets – include only the entries referenced in JSON- # Use file://path entries in config.json when you want to read the value from mounted files instead.- # ACCOUNT_GOOGLE_CLIENT_ID: ''- # ACCOUNT_GOOGLE_CLIENT_SECRET: ''- # ...+ # If you want to override the default configuration mount a config.json as explained below+ # and add the config.json path as an environment variable:+ # CONFIG: /config.jsonports:- '3000:3000'volumes:- server_storage:/var/lib/colanode/storage# Mount a config.json sitting next to this compose file to override defaults.- # If omitted, the already included config.json inside the server image is used.- - ./config.json:/app/apps/server/config.json:ro+ # - ./config.json:/config.json:ronetworks:- colanode_networkdiff --git a/hosting/kubernetes/README.md b/hosting/kubernetes/README.mdindex 152f3768..841d4e2c 100644--- a/hosting/kubernetes/README.md+++ b/hosting/kubernetes/README.md@@ -75,9 +75,9 @@ helm install my-colanode ./hosting/kubernetes/chart \### Using config.json with Helm-- The server image already ships with a default `config.json`. Only two env vars are strictly required: `POSTGRES_URL` and `REDIS_URL` (because the JSON references them via `env://`).-- If you do not override `config.json`, the bundled file still expects those pointers. The chart wires them up automatically via `POSTGRES_URL=env://POSTGRES_URL` and `REDIS_URL=env://REDIS_URL`, so a vanilla install works without extra values.-- To supply your own JSON file, copy `apps/server/config.json`, edit it, and enable the new override:+- The server image already ships with a default configuration. Only two env vars are strictly required: `POSTGRES_URL` and `REDIS_URL` (because the default configuration references them via `env://`).+- If you do add your own `config.json`, the default configuration still expects those pointers. The chart wires them up automatically via `POSTGRES_URL=env://POSTGRES_URL` and `REDIS_URL=env://REDIS_URL`, so a vanilla install works without extra values.+- To supply your own JSON file, copy `apps/server/config.example.json`, edit it, and enable the new override:```bashhelm install my-colanode ./hosting/kubernetes/chart \@@ -87,29 +87,30 @@ helm install my-colanode ./hosting/kubernetes/chart \- Alternatively, create a ConfigMap yourself (`kubectl create configmap colanode-config --from-file=config.json`) and set `colanode.configFile.existingConfigMap=colanode-config`.- Environment variables no longer override config values. Only secrets referenced via `env://` (and values from files via `file://`) are read at runtime. Keep non-secret settings in your JSON, mount it with `colanode.configFile`, and surface additional env vars through `colanode.additionalEnv` when a pointer needs a value from Kubernetes secrets.-- To use `file://` pointers, mount the target files next to `config.json` (the chart stores it at `/app/apps/server/config.json`). For example, to load a PostgreSQL CA cert via `"file://secrets/postgres-ca.crt"`:- 1. Create a secret with the cert contents:-- ```bash- kubectl create secret generic postgres-ca \- --from-file=postgres-ca.crt=./certs/rootCA.crt- ```-- 2. Mount the secret and expose it inside the pod:-- ```yaml- colanode:- extraVolumes:- - name: postgres-ca- secret:- secretName: postgres-ca- extraVolumeMounts:- - name: postgres-ca- mountPath: /app/apps/server/secrets- readOnly: true- ```-- 3. Point your `config.json` field to `"file://secrets/postgres-ca.crt"`. The loader resolves the path relative to the directory containing `config.json`.+- To use `file://` pointers, mount the target files next to `config.json` (the chart stores it at `/config.json`). For example, to load a PostgreSQL CA cert via `"file://secrets/postgres-ca.crt"`:++1. Create a secret with the cert contents:++ ```bash+ kubectl create secret generic postgres-ca \+ --from-file=postgres-ca.crt=./certs/rootCA.crt+ ```++2. Mount the secret and expose it inside the pod:++ ```yaml+ colanode:+ extraVolumes:+ - name: postgres-ca+ secret:+ secretName: postgres-ca+ extraVolumeMounts:+ - name: postgres-ca+ mountPath: /config/secrets+ readOnly: true+ ```++3. Point your `config.json` field to `"file://secrets/postgres-ca.crt"`. The loader resolves the path relative to the directory containing `config.json`.### Storage Configurationdiff --git a/hosting/kubernetes/chart/Chart.yaml b/hosting/kubernetes/chart/Chart.yamlindex ff5fc272..138f807d 100644--- a/hosting/kubernetes/chart/Chart.yaml+++ b/hosting/kubernetes/chart/Chart.yaml@@ -2,7 +2,7 @@ apiVersion: v2name: colanodedescription: A Helm chart for Colanode - open-source & local-first collaboration workspacetype: application-version: 0.2.0+version: 0.2.1# appVersion is auto-updated by the release workflowappVersion: '1.0.0'diff --git a/hosting/kubernetes/chart/templates/_helpers.tpl b/hosting/kubernetes/chart/templates/_helpers.tplindex 2266942b..d5ac3960 100644--- a/hosting/kubernetes/chart/templates/_helpers.tpl+++ b/hosting/kubernetes/chart/templates/_helpers.tpl@@ -170,6 +170,13 @@ Colanode Server Environment Variables- name: REDIS_URLvalue: "redis://:$(REDIS_PASSWORD)@{{ include "colanode.redis.hostname" . }}:6379/0"+{{- $configFile := .Values.colanode.configFile }}+{{- $mountConfigFile := or $configFile.enabled $configFile.existingConfigMap }}+{{- if $mountConfigFile }}+- name: CONFIG+ value: "/config.json"+{{- end }}+{{- range $index, $env := .Values.colanode.additionalEnv }}- name: {{ required (printf "colanode.additionalEnv[%d].name is required" $index) $env.name }}{{- if hasKey $env "valueFrom" }}diff --git a/hosting/kubernetes/chart/templates/deployment.yaml b/hosting/kubernetes/chart/templates/deployment.yamlindex dae8190f..43e01bc8 100644--- a/hosting/kubernetes/chart/templates/deployment.yaml+++ b/hosting/kubernetes/chart/templates/deployment.yaml@@ -61,7 +61,7 @@ spec:{{- end }}{{- if $mountConfigFile }}- name: config-json- mountPath: /app/apps/server/config.json+ mountPath: /config.jsonsubPath: {{ default "config.json" $configFile.key }}readOnly: true{{- end }}diff --git a/hosting/kubernetes/chart/values.yaml b/hosting/kubernetes/chart/values.yamlindex dfcedc09..8222fed1 100644--- a/hosting/kubernetes/chart/values.yaml+++ b/hosting/kubernetes/chart/values.yaml@@ -91,7 +91,7 @@ colanode:# Example: config.json contains "ca": "file://secrets/postgres-ca.crt".# extraVolumeMounts:# - name: postgres-ca- # mountPath: /app/apps/server/secrets+ # mountPath: /config/secrets# readOnly: trueextraVolumeMounts: []