Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

chore(server): customize config merge logic (#11400)

forehalo85d176ce6fad
diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts
index f8e18a297d..18a6aae115 100644
--- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts
+++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts
@@ -64,16 +64,16 @@ test.serial.before(async t => {
copilot: {
providers: {
openai: {
- apiKey: process.env.COPILOT_OPENAI_API_KEY,
+ apiKey: process.env.COPILOT_OPENAI_API_KEY || '',
},
fal: {
- apiKey: process.env.COPILOT_FAL_API_KEY,
+ apiKey: process.env.COPILOT_FAL_API_KEY || '',
},
perplexity: {
- apiKey: process.env.COPILOT_PERPLEXITY_API_KEY,
+ apiKey: process.env.COPILOT_PERPLEXITY_API_KEY || '',
},
gemini: {
- apiKey: process.env.COPILOT_GOOGLE_API_KEY,
+ apiKey: process.env.COPILOT_GOOGLE_API_KEY || '',
},
},
},
diff --git a/packages/backend/server/src/base/config/__tests__/config.spec.ts b/packages/backend/server/src/base/config/__tests__/config.spec.ts
index ac0cc706a8..20001c4b7b 100644
--- a/packages/backend/server/src/base/config/__tests__/config.spec.ts
+++ b/packages/backend/server/src/base/config/__tests__/config.spec.ts
@@ -3,6 +3,7 @@ import test from 'ava';
import { createModule } from '../../../__tests__/create-module';
import { ConfigFactory, ConfigModule } from '..';
import { Config } from '../config';
+import { override } from '../register';
const module = await createModule();
test.after.always(async () => {
@@ -49,13 +50,14 @@ test('should override config', async t => {
auth: {
passwordRequirements: {
max: 10,
+ min: 1,
},
},
});
t.deepEqual(config.auth.passwordRequirements, {
max: 10,
- min: 6,
+ min: 1,
});
});
@@ -88,3 +90,87 @@ Error: Minimum length of password must be less than maximum length`,
}
);
});
+
+test('should override correctly', t => {
+ const config = {
+ auth: {
+ // object config
+ passwordRequirements: {
+ max: 10,
+ min: 6,
+ },
+ allowSignup: false,
+ // keyed config
+ // 'session.ttl', 'session.ttr'
+ session: {
+ ttl: 2000,
+ ttr: 1000,
+ },
+ },
+ storages: {
+ avatar: {
+ // keyed config
+ // "avatar.publicPath: String"
+ publicPath: '/',
+ // object config
+ // "avatar.storage => Object { }"
+ storage: {
+ provider: 'fs',
+ config: {
+ path: '/path/to/avatar',
+ },
+ },
+ },
+ },
+ } as AppConfig;
+
+ override(config, {
+ auth: {
+ passwordRequirements: {
+ max: 20,
+ },
+ allowSignup: true,
+ session: {
+ ttl: 3000,
+ },
+ },
+ storages: {
+ avatar: {
+ storage: {
+ provider: 'aws-s3',
+ config: {
+ credentials: {
+ accessKeyId: '1',
+ accessKeySecret: '1',
+ },
+ },
+ },
+ },
+ },
+ });
+
+ // simple value override
+ t.deepEqual(config.auth.allowSignup, true);
+
+ // right covered left
+ t.deepEqual(config.auth.passwordRequirements, {
+ max: 20,
+ });
+
+ // right merged to left
+ t.deepEqual(config.auth.session, {
+ ttl: 3000,
+ ttr: 1000,
+ });
+
+ // right covered left
+ t.deepEqual(config.storages.avatar.storage, {
+ provider: 'aws-s3',
+ config: {
+ credentials: {
+ accessKeyId: '1',
+ accessKeySecret: '1',
+ },
+ },
+ });
+});
diff --git a/packages/backend/server/src/base/config/factory.ts b/packages/backend/server/src/base/config/factory.ts
index ad6c7a8877..82f2673d21 100644
--- a/packages/backend/server/src/base/config/factory.ts
+++ b/packages/backend/server/src/base/config/factory.ts
@@ -1,14 +1,13 @@
import { Inject, Injectable, Optional } from '@nestjs/common';
-import { merge } from 'lodash-es';
import { InvalidAppConfig } from '../error';
-import { APP_CONFIG_DESCRIPTORS, getDefaultConfig } from './register';
+import { APP_CONFIG_DESCRIPTORS, getDefaultConfig, override } from './register';
export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN');
@Injectable()
export class ConfigFactory {
- readonly #config: DeepReadonly<AppConfig>;
+ readonly #config: AppConfig;
constructor(
@Inject(OVERRIDE_CONFIG_TOKEN)
@@ -22,8 +21,12 @@ export class ConfigFactory {
return this.#config;
}
+ clone() {
+ return structuredClone(this.#config);
+ }
+
override(updates: DeepPartial<AppConfig>) {
- merge(this.#config, updates);
+ override(this.#config, updates);
}
validate(updates: Array<{ module: string; key: string; value: any }>) {
@@ -53,8 +56,9 @@ Error: ${issue.message}`);
}
}
- private loadDefault(): DeepReadonly<AppConfig> {
+ private loadDefault() {
const config = getDefaultConfig();
- return merge(config, this.overrides);
+ override(config, this.overrides);
+ return config;
}
}
diff --git a/packages/backend/server/src/base/config/register.ts b/packages/backend/server/src/base/config/register.ts
index 9d203c5a4d..28a329804b 100644
--- a/packages/backend/server/src/base/config/register.ts
+++ b/packages/backend/server/src/base/config/register.ts
@@ -1,7 +1,8 @@
import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
import { join } from 'node:path';
-import { merge, once, set } from 'lodash-es';
+import { mergeWith, once, set } from 'lodash-es';
import { z } from 'zod';
import { type EnvConfigType, parseEnvValue } from './env';
@@ -208,14 +209,15 @@ export function defineModuleConfig<T extends keyof AppConfigSchema>(
};
}
-function readConfigJSONOverrides() {
- if (existsSync(join(env.projectRoot, 'config.json'))) {
+const CONFIG_JSON_PATHS = [
+ join(env.projectRoot, 'config.json'),
+ `${homedir()}/.affine/config/config.json`,
+];
+function readConfigJSONOverrides(path: string) {
+ const overrides: DeepPartial<AppConfig> = {};
+ if (existsSync(path)) {
try {
- const config = JSON.parse(
- readFileSync(join(env.projectRoot, 'config.json'), 'utf-8')
- ) as AppConfig;
-
- const overrides = {};
+ const config = JSON.parse(readFileSync(path, 'utf-8')) as AppConfig;
Object.entries(config).forEach(([key, value]) => {
if (key === '$schema') {
@@ -226,18 +228,41 @@ function readConfigJSONOverrides() {
set(overrides, `${key}.${k}`, v);
});
});
-
- return overrides;
} catch (e) {
console.error('Invalid json config file', e);
}
}
- return {};
+ return overrides;
}
-export function getDefaultConfig(): AppConfigSchema {
- const config: Record<string, any> = {};
+export function override(config: AppConfig, update: DeepPartial<AppConfig>) {
+ Object.keys(update).forEach(module => {
+ const moduleDescriptors = APP_CONFIG_DESCRIPTORS[module];
+ const configKeys = new Set(Object.keys(moduleDescriptors));
+
+ const moduleConfig = config[module as keyof AppConfig];
+ const moduleOverrides = update[module as keyof AppConfig];
+
+ const merge = (left: any, right: any, path: string = '') => {
+ // if we found the key in the config keys
+ // we should use the override object instead of merge it with left
+ if (configKeys.has(path)) {
+ return right;
+ }
+
+ // go deeper
+ return mergeWith(left, right, (left, right, key) => {
+ return merge(left, right, path === '' ? key : `${path}.${key}`);
+ });
+ };
+
+ config[module as keyof AppConfig] = merge(moduleConfig, moduleOverrides);
+ });
+}
+
+export function getDefaultConfig(): AppConfig {
+ const config = {} as AppConfig;
const envs = process.env;
for (const [module, defs] of Object.entries(APP_CONFIG_DESCRIPTORS)) {
@@ -271,12 +296,14 @@ Error: ${issue.message}`;
set(modulizedConfig, key, defaultValue);
}
+ // @ts-expect-error all keys are known
config[module] = modulizedConfig;
}
- const fileOverrides = readConfigJSONOverrides();
-
- merge(config, fileOverrides);
+ CONFIG_JSON_PATHS.forEach(path => {
+ const overrides = readConfigJSONOverrides(path);
+ override(config, overrides);
+ });
return config as AppConfigSchema;
}
diff --git a/packages/backend/server/src/base/event/eventbus.ts b/packages/backend/server/src/base/event/eventbus.ts
index a3ea945e4a..af0c33a01a 100644
--- a/packages/backend/server/src/base/event/eventbus.ts
+++ b/packages/backend/server/src/base/event/eventbus.ts
@@ -155,7 +155,7 @@ export class EventBus
add.call(this.emitter, event, handler as any, opts);
- this.logger.verbose(`Event handler registered ${signature}`);
+ this.logger.log(`Event handler registered ${signature}`);
return () => {
this.emitter.off(event, handler as any);
diff --git a/packages/backend/server/src/base/job/queue/scanner.ts b/packages/backend/server/src/base/job/queue/scanner.ts
index cf53a053da..52ecec5782 100644
--- a/packages/backend/server/src/base/job/queue/scanner.ts
+++ b/packages/backend/server/src/base/job/queue/scanner.ts
@@ -1,4 +1,4 @@
-import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { ModuleScanner } from '../../nestjs';
import { getJobHandlerMetadata, JOB_SIGNAL } from './def';
@@ -11,6 +11,7 @@ interface JobHandler {
@Injectable()
export class JobHandlerScanner implements OnModuleInit {
private readonly handlers: Record<string, JobHandler> = {};
+ private readonly logger = new Logger(JobHandlerScanner.name);
constructor(private readonly scanner: ModuleScanner) {}
@@ -70,6 +71,8 @@ export class JobHandlerScanner implements OnModuleInit {
return instance[method].bind(instance)(payload);
},
};
+
+ this.logger.log(`Job handler registered [${jobName}] (${signature})`);
});
});
});
diff --git a/packages/backend/server/src/core/config/__tests__/service.spec.ts b/packages/backend/server/src/core/config/__tests__/service.spec.ts
index 5c0bcb4158..6947113139 100644
--- a/packages/backend/server/src/core/config/__tests__/service.spec.ts
+++ b/packages/backend/server/src/core/config/__tests__/service.spec.ts
@@ -23,7 +23,7 @@ test.after.always(async () => {
});
test('should update config', async t => {
- const oldValue = service.config.server.externalUrl;
+ const oldValue = service.getConfig().server.externalUrl;
const newValue = faker.internet.url();
await service.updateConfig(user.id, [
{
@@ -33,8 +33,8 @@ test('should update config', async t => {
},
]);
- t.not(service.config.server.externalUrl, oldValue);
- t.is(service.config.server.externalUrl, newValue);
+ t.not(service.getConfig().server.externalUrl, oldValue);
+ t.is(service.getConfig().server.externalUrl, newValue);
});
test('should validate config before update', async t => {
@@ -53,7 +53,7 @@ Error: Invalid url`,
}
);
- t.not(service.config.server.externalUrl, 'invalid-url');
+ t.not(service.getConfig().server.externalUrl, 'invalid-url');
await t.throwsAsync(
service.updateConfig(user.id, [
@@ -68,8 +68,11 @@ Error: Invalid url`,
}
);
- // @ts-expect-error allow
- t.is(service.config.auth['unknown-key'], undefined);
+ t.is(
+ // @ts-expect-error testing
+ service.getConfig().auth['unknown-key'],
+ undefined
+ );
});
test('should emit config.init event', async t => {
@@ -77,12 +80,12 @@ test('should emit config.init event', async t => {
const event = module.event.last('config.init');
t.is(event.name, 'config.init');
t.deepEqual(event.payload, {
- config: service.config,
+ config: service.getConfig(),
});
});
test('should revalidate config', async t => {
- const outdatedValue = service.config.server.externalUrl;
+ const outdatedValue = service.getConfig().server.externalUrl;
const newValue = faker.internet.url();
await models.appConfig.save(user.id, [
@@ -94,8 +97,8 @@ test('should revalidate config', async t => {
await service.revalidateConfig();
- t.not(service.config.server.externalUrl, outdatedValue);
- t.is(service.config.server.externalUrl, newValue);
+ t.not(service.getConfig().server.externalUrl, outdatedValue);
+ t.is(service.getConfig().server.externalUrl, newValue);
});
test('should emit config changed event', async t => {
diff --git a/packages/backend/server/src/core/config/resolver.ts b/packages/backend/server/src/core/config/resolver.ts
index a76fa0bb38..05ea8356db 100644
--- a/packages/backend/server/src/core/config/resolver.ts
+++ b/packages/backend/server/src/core/config/resolver.ts
@@ -191,7 +191,7 @@ export class AppConfigResolver {
description: 'get the whole app configuration',
})
appConfig() {
- return this.service.config;
+ return this.service.getConfig();
}
@Mutation(() => GraphQLJSONObject, {
diff --git a/packages/backend/server/src/core/config/service.ts b/packages/backend/server/src/core/config/service.ts
index 3fa9938759..b8634b0bb0 100644
--- a/packages/backend/server/src/core/config/service.ts
+++ b/packages/backend/server/src/core/config/service.ts
@@ -56,8 +56,8 @@ export class ServerService implements OnApplicationBootstrap {
this.#features.delete(feature);
}
- get config() {
- return this.configFactory.config;
+ getConfig() {
+ return this.configFactory.clone();
}
async updateConfig(