Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
perf(plugins): track plugin storage use as a running total
Yudhi Armyndharis8aba36b7419f
diff --git a/src/core/plugins/plugin-storage.service.spec.ts b/src/core/plugins/plugin-storage.service.spec.tsindex 3df99f401..06d111b8a 100644--- a/src/core/plugins/plugin-storage.service.spec.ts+++ b/src/core/plugins/plugin-storage.service.spec.ts@@ -283,6 +283,60 @@ describe('PluginStorageService per-plugin storage quota', () => {/storage quota exceeded.*max 52428800\)/,);});++ describe('running total', () => {+ const pluginDir = (id = 'quota-plugin'): string => path.join(dataDir, 'plugins', id);++ it('reads the directory once, not on every write', async () => {+ const storage = makeStorage(10 * 1024 * 1024);+ const readdir = jest.spyOn(fs, 'readdirSync');+ try {+ for (let i = 0; i < 50; i++) await storage.set(`key-${i}`, 'x'.repeat(100));+ expect(readdir.mock.calls.filter(([dir]) => String(dir) === pluginDir())).toHaveLength(1);+ } finally {+ readdir.mockRestore();+ }+ });++ it('frees a deleted key and counts a replaced key at its new size', async () => {+ const storage = makeStorage(1024);+ await storage.set('a', 'x'.repeat(500)); // 502 bytes+ await storage.set('b', 'y'.repeat(400)); // 402 bytes+ await expect(storage.set('c', 'z'.repeat(300))).rejects.toThrow(/storage quota exceeded/);++ await storage.delete('a');+ await expect(storage.set('c', 'z'.repeat(300))).resolves.toBeUndefined(); // 402 + 302++ await storage.set('b', 'y'.repeat(10)); // 402 -> 12 bytes+ await expect(storage.set('d', 'w'.repeat(600))).resolves.toBeUndefined(); // 302 + 12 + 602+ await expect(storage.set('e', 'v'.repeat(300))).rejects.toThrow(/storage quota exceeded/);+ });++ it('recounts from the directory before refusing, so files removed behind its back do not block writes', async () => {+ const storage = makeStorage(1024);+ await storage.set('a', 'x'.repeat(900)); // 902 bytes++ for (const file of fs.readdirSync(pluginDir())) fs.unlinkSync(path.join(pluginDir(), file));++ await expect(storage.set('b', 'y'.repeat(900))).resolves.toBeUndefined();+ // The recount did not forget the new key: a third write that cannot fit is still refused.+ await expect(storage.set('c', 'z'.repeat(300))).rejects.toThrow(/storage quota exceeded/);+ });++ it('starts again from zero once the plugin data is deleted', async () => {+ const configService = {+ get: (k: string) => (k === 'dataDir' ? dataDir : k === 'plugins.storageMaxBytes' ? 1024 : undefined),+ } as unknown as ConfigService;+ const service = new PluginStorageService(configService);+ const first = service.createPluginStorage('reinstalled');+ await first.set('state', 'x'.repeat(900));++ service.deletePluginData('reinstalled');+ const second = service.createPluginStorage('reinstalled');++ await expect(second.set('state', 'y'.repeat(900))).resolves.toBeUndefined();+ });+ });});describe('PluginStorageService unreadable registry', () => {diff --git a/src/core/plugins/plugin-storage.service.ts b/src/core/plugins/plugin-storage.service.tsindex 8ab9388e6..aaba8d64b 100644--- a/src/core/plugins/plugin-storage.service.ts+++ b/src/core/plugins/plugin-storage.service.ts@@ -66,6 +66,8 @@ export class PluginStorageService {private readonly dataDir: string;private readonly registryPath: string;private readonly maxPluginStorageBytes: number;+ /** Bytes in each plugin's encoded key files, once a write or delete has seeded it. See assertStorageQuota. */+ private readonly storageUsage = new Map<string, number>();private registry: Map<string, PluginRegistryEntry> = new Map();constructor(private readonly configService: ConfigService) {@@ -256,6 +258,7 @@ export class PluginStorageService {}try {fs.rmSync(dir, { recursive: true, force: true });+ this.storageUsage.delete(pluginId);this.logger.debug(`Deleted plugin storage: ${pluginId}`, { pluginId, action: 'plugin_storage_deleted' });} catch (error) {this.logger.warn(`Failed to delete plugin storage for ${pluginId}`, {@@ -273,17 +276,29 @@ export class PluginStorageService {* package files must not eat its quota. The file being overwritten is excluded from the current* usage so replacing a large key does not count its bytes twice.*/- private assertStorageQuota(pluginDataDir: string, targetPath: string, incomingBytes: number, pluginId: string): void {- let used = 0;- for (const entry of fs.readdirSync(pluginDataDir)) {- if (!entry.startsWith(ENCODED_KEY_PREFIX) || !entry.endsWith('.json')) continue;- const full = path.join(pluginDataDir, entry);- if (full === targetPath) continue;- try {- used += fs.statSync(full).size;- } catch {- /* vanished between readdir and stat; ignore */- }+ private assertStorageQuota(+ pluginDataDir: string,+ targetPath: string,+ incomingBytes: number,+ pluginId: string,+ ): number {+ let replaced = 0;+ try {+ replaced = fs.statSync(targetPath).size;+ } catch {+ /* a new key */+ }+ // The running total is seeded by one scan and kept in step by set/delete, so a write costs one stat+ // instead of a read of the whole directory: that read ran synchronously on the host's main thread for+ // every write and grew with the number of keys the plugin had stored (about 50 ms per write at 20,000).+ let total = this.storageUsage.get(pluginId) ?? this.scanStorageBytes(pluginDataDir);+ let used = Math.max(0, total - replaced);+ if (used + incomingBytes > this.maxPluginStorageBytes) {+ // Files removed behind the service's back leave the total too high; confirm against the directory+ // before refusing the write.+ total = this.scanStorageBytes(pluginDataDir);+ used = Math.max(0, total - replaced);+ this.storageUsage.set(pluginId, total);}if (used + incomingBytes > this.maxPluginStorageBytes) {throw new Error(@@ -292,6 +307,27 @@ export class PluginStorageService {} bytes (max ${this.maxPluginStorageBytes}). Delete unused keys or raise PLUGIN_STORAGE_MAX_BYTES.`,);}+ return used;+ }++ /** Bytes held by the plugin's encoded key files, read from the directory. */+ private scanStorageBytes(pluginDataDir: string): number {+ let total = 0;+ for (const entry of fs.readdirSync(pluginDataDir)) {+ if (!entry.startsWith(ENCODED_KEY_PREFIX) || !entry.endsWith('.json')) continue;+ try {+ total += fs.statSync(path.join(pluginDataDir, entry)).size;+ } catch {+ /* vanished between readdir and stat; ignore */+ }+ }+ return total;+ }++ /** Take a removed key's bytes out of the running total, if one has been seeded. */+ private releaseStorageBytes(pluginId: string, bytes: number): void {+ const total = this.storageUsage.get(pluginId);+ if (total !== undefined) this.storageUsage.set(pluginId, Math.max(0, total - bytes));}createPluginStorage(pluginId: string): PluginStorage {@@ -354,12 +390,14 @@ export class PluginStorageService {}try {const payload = JSON.stringify(value, null, 2);- this.assertStorageQuota(pluginDataDir, filePath, Buffer.byteLength(payload, 'utf8'), pluginId);+ const bytes = Buffer.byteLength(payload, 'utf8');+ const usedByOtherKeys = this.assertStorageQuota(pluginDataDir, filePath, bytes, pluginId);// 0o600 (owner-only): a plugin-persisted secret must not land in a group/other-readable file.// The mode on the temp write carries through the rename; chmod is a backstop if the target// pre-existed (writeFileSync mode only applies on create). Mirrors saveRegistry's hardening.atomicWriteFileSync(filePath, payload, { mode: 0o600 });fs.chmodSync(filePath, 0o600);+ this.storageUsage.set(pluginId, usedByOtherKeys + bytes);// Keep any legacy file in place. Encoded-first reads and de-duplicated lists ensure it cannot// shadow this write, while avoiding an unlink outside the service-owned key-* namespace.return Promise.resolve();@@ -379,7 +417,10 @@ export class PluginStorageService {const candidates = legacyPath && legacyPath !== filePath ? [filePath, legacyPath] : [filePath];for (const candidate of candidates) {if (fs.existsSync(candidate)) {+ // Only the encoded file counts toward the quota; a legacy file is never in the total.+ const bytes = candidate === filePath ? fs.statSync(candidate).size : 0;fs.unlinkSync(candidate);+ this.releaseStorageBytes(pluginId, bytes);}}return Promise.resolve();