A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(user): stop double-writing last active on every request (#2148)
Hampuse83a2d6aec47
diff --git a/fluxer_api/src/api/admin/tests/AdminLastActiveIpSearch.test.ts b/fluxer_api/src/api/admin/tests/AdminLastActiveIpSearch.test.tsindex ce59f33a3..ce30f0dd3 100644--- a/fluxer_api/src/api/admin/tests/AdminLastActiveIpSearch.test.ts+++ b/fluxer_api/src/api/admin/tests/AdminLastActiveIpSearch.test.ts@@ -2,6 +2,7 @@import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest';import {createTestAccount, setUserACLs} from '../../auth/tests/AuthTestUtils';+import {getUserActivityBuffer} from '../../middleware/ServiceSingletons';import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';import {HTTP_STATUS} from '../../test/TestConstants';import {createBuilder} from '../../test/TestRequestBuilder';@@ -19,6 +20,7 @@ async function setLastActiveIp(harness: ApiTestHarness, token: string, ip: strin.header('x-forwarded-for', ip).expect(HTTP_STATUS.OK).execute();+ await getUserActivityBuffer().drainAndFlush();}describe('Admin last active IP search', () => {diff --git a/fluxer_api/src/api/admin/tests/AdminSearchEndpoints.test.ts b/fluxer_api/src/api/admin/tests/AdminSearchEndpoints.test.tsindex 5b8ff92d6..9f3be9ae4 100644--- a/fluxer_api/src/api/admin/tests/AdminSearchEndpoints.test.ts+++ b/fluxer_api/src/api/admin/tests/AdminSearchEndpoints.test.ts@@ -3,6 +3,7 @@import {afterEach, beforeEach, describe, expect, test} from 'vitest';import {createTestAccount, setUserACLs} from '../../auth/tests/AuthTestUtils';import {createDmChannel, createFriendship, createGuild} from '../../channel/tests/ChannelTestUtils';+import {getUserActivityBuffer} from '../../middleware/ServiceSingletons';import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';import {HTTP_STATUS} from '../../test/TestConstants';import {createBuilder} from '../../test/TestRequestBuilder';@@ -13,6 +14,7 @@ async function setLastActiveIp(harness: ApiTestHarness, token: string, ip: strin.header('x-forwarded-for', ip).expect(HTTP_STATUS.OK).execute();+ await getUserActivityBuffer().drainAndFlush();}describe('Admin Search Endpoints', () => {diff --git a/fluxer_api/src/api/admin/tests/AdminSearchFieldCoverage.test.ts b/fluxer_api/src/api/admin/tests/AdminSearchFieldCoverage.test.tsindex 958b0ff6a..376de5d38 100644--- a/fluxer_api/src/api/admin/tests/AdminSearchFieldCoverage.test.ts+++ b/fluxer_api/src/api/admin/tests/AdminSearchFieldCoverage.test.ts@@ -5,6 +5,7 @@ import type {UserAdminResponse} from '@fluxer/schema/src/domains/admin/AdminUserimport {afterEach, beforeEach, describe, expect, test} from 'vitest';import {createTestAccount, setUserACLs} from '../../auth/tests/AuthTestUtils';import {createGuild} from '../../channel/tests/ChannelTestUtils';+import {getUserActivityBuffer} from '../../middleware/ServiceSingletons';import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';import {HTTP_STATUS} from '../../test/TestConstants';import {createBuilder, createBuilderWithoutAuth} from '../../test/TestRequestBuilder';@@ -40,6 +41,7 @@ async function setLastActiveIp(harness: ApiTestHarness, token: string, ip: strin.header('x-forwarded-for', ip).expect(HTTP_STATUS.OK).execute();+ await getUserActivityBuffer().drainAndFlush();}describe('Admin Search Field Coverage', () => {diff --git a/fluxer_api/src/api/auth/AuthSession.ts b/fluxer_api/src/api/auth/AuthSession.tsindex 707f7e60e..5435bd885 100644--- a/fluxer_api/src/api/auth/AuthSession.ts+++ b/fluxer_api/src/api/auth/AuthSession.ts@@ -35,15 +35,6 @@ interface LogoutAuthSessionsParams {sessionIdHashes: Array<string>;}-interface UpdateUserActivityParams {- userId: UserID;- clientIp: string;- user?: User;- action?: 'session_authenticated' | 'bearer_fallback_session_authenticated' | 'unknown';- tokenType?: 'session' | 'bearer';- sessionId?: string;-}-interface DispatchAuthSessionChangeParams {userId: UserID;oldAuthSessionIdHash: string;@@ -153,11 +144,6 @@ export async function updateAuthSessionLastUsed(ctx: ApiContext, tokenHash: Uintawait ctx.services.userActivityBuffer.recordAuthSessionActivity(Buffer.from(tokenHash), new Date());}-export async function updateUserActivity(ctx: ApiContext, {userId, clientIp}: UpdateUserActivityParams): Promise<void> {- const {users} = ctx.services;- await users.updateUserActivity(userId, clientIp);-}-export async function revokeToken(ctx: ApiContext, token: string): Promise<void> {const {users, gateway} = ctx.services;const tokenHash = Buffer.from(AuthUtility.getTokenIdHash(ctx, token));diff --git a/fluxer_api/src/api/middleware/UserMiddleware.ts b/fluxer_api/src/api/middleware/UserMiddleware.tsindex 570520c4d..0814a6d27 100644--- a/fluxer_api/src/api/middleware/UserMiddleware.ts+++ b/fluxer_api/src/api/middleware/UserMiddleware.ts@@ -112,17 +112,6 @@ export const UserMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {if (authSession) {void AuthSession.updateAuthSessionLastUsed(apiContext, authSession.sessionIdHash);const user = await apiContext.services.users.findUniqueAssert(authSession.userId);- const sessionId = Buffer.from(authSession.sessionIdHash).toString('base64url');- void AuthSession.updateUserActivity(apiContext, {- userId: authSession.userId,- clientIp: resolvedClientIp,- user,- action: 'session_authenticated',- tokenType: 'session',- sessionId,- }).catch((error: unknown) => {- Logger.warn({error, userId: authSession.userId}, 'Failed to update user activity telemetry');- });ctx.set('authSession', authSession);ctx.set('authTokenType', 'session');setUserInContext(ctx, user, true);diff --git a/fluxer_api/src/api/user/repositories/IUserAuthRepository.ts b/fluxer_api/src/api/user/repositories/IUserAuthRepository.tsindex ea4d59e7e..59f24752e 100644--- a/fluxer_api/src/api/user/repositories/IUserAuthRepository.ts+++ b/fluxer_api/src/api/user/repositories/IUserAuthRepository.ts@@ -44,7 +44,6 @@ export interface IUserAuthRepository {createPhoneToken(token: PhoneVerificationToken, phone: string, userId: UserID | null): Promise<void>;getPhoneToken(token: PhoneVerificationToken): Promise<PhoneTokenRow | null>;deletePhoneToken(token: PhoneVerificationToken): Promise<void>;- updateUserActivity(userId: UserID, clientIp: string): Promise<void>;checkIpAuthorized(userId: UserID, ip: string): Promise<boolean>;createAuthorizedIp(userId: UserID, ip: string): Promise<void>;createIpAuthorizationToken(userId: UserID, token: string, email: string): Promise<void>;diff --git a/fluxer_api/src/api/user/repositories/UserAuthRepository.ts b/fluxer_api/src/api/user/repositories/UserAuthRepository.tsindex c7b15ea06..26d84f3b5 100644--- a/fluxer_api/src/api/user/repositories/UserAuthRepository.ts+++ b/fluxer_api/src/api/user/repositories/UserAuthRepository.ts@@ -153,10 +153,6 @@ export class UserAuthRepository implements IUserAuthRepository {return this.tokenRepository.deletePhoneToken(token);}- async updateUserActivity(userId: UserID, clientIp: string): Promise<void> {- return this.ipAuthorizationRepository.updateUserActivity(userId, clientIp);- }-async checkIpAuthorized(userId: UserID, ip: string): Promise<boolean> {return this.ipAuthorizationRepository.checkIpAuthorized(userId, ip);}diff --git a/fluxer_api/src/api/user/repositories/UserRepository.ts b/fluxer_api/src/api/user/repositories/UserRepository.tsindex 67ee137c5..9cb3703ee 100644--- a/fluxer_api/src/api/user/repositories/UserRepository.ts+++ b/fluxer_api/src/api/user/repositories/UserRepository.ts@@ -348,10 +348,6 @@ export class UserRepository implements IUserRepositoryAggregate {return this.authRepo.deletePhoneToken(token);}- async updateUserActivity(userId: UserID, clientIp: string): Promise<void> {- return this.authRepo.updateUserActivity(userId, clientIp);- }-async checkIpAuthorized(userId: UserID, ip: string): Promise<boolean> {return this.authRepo.checkIpAuthorized(userId, ip);}diff --git a/fluxer_api/src/api/user/repositories/account/crud/UserDataRepository.ts b/fluxer_api/src/api/user/repositories/account/crud/UserDataRepository.tsindex fd0da4eee..34ca02e79 100644--- a/fluxer_api/src/api/user/repositories/account/crud/UserDataRepository.ts+++ b/fluxer_api/src/api/user/repositories/account/crud/UserDataRepository.ts@@ -174,6 +174,7 @@ export class UserDataRepository {};}> {const {userId, lastActiveAt, lastActiveIp} = params;+ const previousData = (await this.getActivityTracking(userId)) ?? {last_active_at: null, last_active_ip: null};await upsertOne(Users.patchByPk({user_id: userId},@@ -184,7 +185,7 @@ export class UserDataRepository {),);return {- previousData: {last_active_at: null, last_active_ip: null},+ previousData,updatedData: {last_active_at: lastActiveAt, last_active_ip: lastActiveIp ?? null},};}diff --git a/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.ts b/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.tsindex 114c04557..832c035ae 100644--- a/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.ts+++ b/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.ts@@ -163,7 +163,7 @@ export class UserIndexRepository {);}}- await batch.execute();+ await batch.execute(false);}async deleteIndices(diff --git a/fluxer_api/src/api/user/repositories/auth/IpAuthorizationRepository.ts b/fluxer_api/src/api/user/repositories/auth/IpAuthorizationRepository.tsindex b13e6d06e..9b7556c8a 100644--- a/fluxer_api/src/api/user/repositories/auth/IpAuthorizationRepository.ts+++ b/fluxer_api/src/api/user/repositories/auth/IpAuthorizationRepository.ts@@ -111,15 +111,6 @@ export class IpAuthorizationRepository {return {userId: result.user_id, email: result.email};}- async updateUserActivity(userId: UserID, clientIp: string): Promise<void> {- const now = new Date();- await this.userAccountRepository.updateLastActiveAt({- userId,- lastActiveAt: now,- lastActiveIp: clientIp,- });- }-async getAuthorizedIps(userId: UserID): Promise<Array<{ip: string;diff --git a/fluxer_api/src/api/user/services/UserActivityBuffer.ts b/fluxer_api/src/api/user/services/UserActivityBuffer.tsindex ba2112d72..6c240ca3c 100644--- a/fluxer_api/src/api/user/services/UserActivityBuffer.ts+++ b/fluxer_api/src/api/user/services/UserActivityBuffer.ts@@ -6,8 +6,9 @@ import type {UserID} from '../../BrandedTypes';import {upsertOne} from '../../database/CassandraQueryExecution';import {Db} from '../../database/CassandraTypes';import {Logger} from '../../Logger';-import {AuthSessions, Users} from '../../Tables';+import {AuthSessions} from '../../Tables';import {isJsonRecord, parseJsonRecord} from '../../utils/JsonBoundaryUtils';+import {UserAccountRepository} from '../repositories/account/UserAccountRepository';const PENDING_HASH_KEY = 'user_activity:pending';const PENDING_AUTH_SESSION_HASH_KEY = 'auth_session_activity:pending';@@ -16,6 +17,10 @@ const WRITE_CONCURRENCY = 64;const AUTH_SESSION_TOUCH_DEBOUNCE_TTL_SECONDS = seconds('5 minutes');type ActivityWriter = typeof upsertOne;+interface UserActivityAccountWriter {+ updateLastActiveAt(params: {userId: UserID; lastActiveAt: Date; lastActiveIp?: string}): Promise<void>;+}+interface PendingEntry {ts: number;ip: string | null;@@ -58,10 +63,16 @@ function isStringRecord(value: unknown): value is Record<string, string> {export class UserActivityBuffer {private readonly kv: IKVProvider;private readonly writer: ActivityWriter;+ private readonly accounts: UserActivityAccountWriter;- constructor(kv: IKVProvider, writer: ActivityWriter = upsertOne) {+ constructor(+ kv: IKVProvider,+ writer: ActivityWriter = upsertOne,+ accounts: UserActivityAccountWriter = new UserAccountRepository(kv),+ ) {this.kv = kv;this.writer = writer;+ this.accounts = accounts;}recordActivity(userId: UserID, timestamp: Date, ip: string | null): void {@@ -128,15 +139,11 @@ export class UserActivityBuffer {const chunk = drained.slice(i, i + WRITE_CONCURRENCY);const results = await Promise.allSettled(chunk.map(({userId, entry}) =>- this.writer(- Users.patchByPk(- {user_id: userId},- {- last_active_at: Db.set(new Date(entry.ts)),- last_active_ip: entry.ip !== null ? Db.set(entry.ip) : Db.clear(),- },- ),- ),+ this.accounts.updateLastActiveAt({+ userId,+ lastActiveAt: new Date(entry.ts),+ lastActiveIp: entry.ip ?? undefined,+ }),),);for (const r of results) {diff --git a/fluxer_api/src/api/user/tests/UserActivityFlush.test.ts b/fluxer_api/src/api/user/tests/UserActivityFlush.test.tsnew file mode 100644index 000000000..0660068ae--- /dev/null+++ b/fluxer_api/src/api/user/tests/UserActivityFlush.test.ts@@ -0,0 +1,53 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest';+import {createTestAccount} from '../../auth/tests/AuthTestUtils';+import {createUserID} from '../../BrandedTypes';+import {getUserActivityBuffer, getUserRepository} from '../../middleware/ServiceSingletons';+import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';+import {HTTP_STATUS} from '../../test/TestConstants';+import {createBuilder} from '../../test/TestRequestBuilder';++const REGISTRATION_IP = '198.51.100.7';+const REQUEST_IP = '203.0.113.7';++async function fetchMeFromIp(harness: ApiTestHarness, token: string, ip: string): Promise<void> {+ await createBuilder(harness, token).get('/users/@me').header('x-forwarded-for', ip).expect(HTTP_STATUS.OK).execute();+}++describe('User activity buffering', () => {+ let harness: ApiTestHarness;+ beforeAll(async () => {+ harness = await createApiTestHarness();+ });+ beforeEach(async () => {+ await harness.reset();+ });+ afterAll(async () => {+ await harness?.shutdown();+ });+ test('an authenticated request buffers last active instead of writing it', async () => {+ const account = await createTestAccount(harness, {ipAddress: REGISTRATION_IP});+ const userId = createUserID(BigInt(account.userId));+ await fetchMeFromIp(harness, account.token, REQUEST_IP);+ const pending = await harness.kvProvider.hgetall('user_activity:pending');+ expect(pending[account.userId]).toBeDefined();+ const beforeFlush = await getUserRepository().getActivityTracking(userId);+ expect(beforeFlush?.last_active_ip).toBe(REGISTRATION_IP);+ await getUserActivityBuffer().drainAndFlush();+ const afterFlush = await getUserRepository().getActivityTracking(userId);+ expect(afterFlush?.last_active_ip).toBe(REQUEST_IP);+ });+ test('flushing moves the last active IP index and prunes the previous address', async () => {+ const account = await createTestAccount(harness, {ipAddress: REGISTRATION_IP});+ const userRepository = getUserRepository();+ const seeded = await userRepository.listUserIdsByLastActiveIp(REGISTRATION_IP, 10, 0);+ expect(seeded.userIds.map((id) => id.toString())).toContain(account.userId);+ await fetchMeFromIp(harness, account.token, REQUEST_IP);+ await getUserActivityBuffer().drainAndFlush();+ const previous = await userRepository.listUserIdsByLastActiveIp(REGISTRATION_IP, 10, 0);+ expect(previous.userIds.map((id) => id.toString())).not.toContain(account.userId);+ const current = await userRepository.listUserIdsByLastActiveIp(REQUEST_IP, 10, 0);+ expect(current.userIds.map((id) => id.toString())).toContain(account.userId);+ });+});