A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(api/admin): search users by ACL through an ACL index
Hampus556be21a2f8a
diff --git a/fluxer_api/src/api/admin/services/AdminSearchService.ts b/fluxer_api/src/api/admin/services/AdminSearchService.tsindex b87e887fb..1a0afac3d 100644--- a/fluxer_api/src/api/admin/services/AdminSearchService.ts+++ b/fluxer_api/src/api/admin/services/AdminSearchService.ts@@ -38,6 +38,11 @@ interface AdminSearchServiceDeps {auditService: AdminAuditService;}+function compareUserIds(a: UserID, b: UserID): number {+ if (a === b) return 0;+ return a < b ? -1 : 1;+}+export class AdminSearchService {constructor(private readonly deps: AdminSearchServiceDeps) {}@@ -94,10 +99,12 @@ export class AdminSearchService {query?: string;email?: string;last_active_ip?: string;+ acl?: string;limit: number;offset: number;},acls: ReadonlySet<string>,+ locale?: string,) {const {users: userRepository, cache: cacheService} = this.deps.apiContext.services;const email = data.email?.trim();@@ -106,18 +113,25 @@ export class AdminSearchService {if (!user) {return {users: [], total: 0};}- const response = await mapUserToAdminResponse(user, cacheService, acls);+ const response = await mapUserToAdminResponse(user, cacheService, acls, locale);return {users: [response], total: 1};}const lastActiveIp = data.last_active_ip?.trim();if (lastActiveIp) {const {userIds, total} = await userRepository.listUserIdsByLastActiveIp(lastActiveIp, data.limit, data.offset);- const response = await this.mapOrderedUsers(userIds, acls);+ const response = await this.mapOrderedUsers(userIds, acls, locale);return {users: response,total,};}+ if (data.acl !== undefined) {+ const userIds = (await userRepository.listUserIdsWithAcl(data.acl))+ .filter((userId) => !isSyntheticUserId(userId))+ .sort(compareUserIds);+ const response = await this.mapOrderedUsers(userIds.slice(data.offset, data.offset + data.limit), acls, locale);+ return {users: response, total: userIds.length};+ }const userSearchService = getUserSearchService();if (!userSearchService) {throw new FeatureTemporarilyDisabledError();@@ -135,11 +149,11 @@ export class AdminSearchService {]);const {hits, total} = searchResult;const userIds = hits.map((hit) => createUserID(BigInt(hit.id)));- const response = await this.mapOrderedUsers(userIds, acls);+ const response = await this.mapOrderedUsers(userIds, acls, locale);if (directUser && data.offset === 0) {const directId = directUser.id.toString();if (!response.some((u) => u.id === directId)) {- response.unshift(await mapUserToAdminResponse(directUser, cacheService, acls));+ response.unshift(await mapUserToAdminResponse(directUser, cacheService, acls, locale));}}return {@@ -250,7 +264,7 @@ export class AdminSearchService {return status;}- private async mapOrderedUsers(userIds: Array<UserID>, acls: ReadonlySet<string>) {+ private async mapOrderedUsers(userIds: Array<UserID>, acls: ReadonlySet<string>, locale?: string) {const {users: userRepository, cache: cacheService} = this.deps.apiContext.services;const users = await userRepository.listUsers(userIds);const usersById = new Map(users.map((user) => [user.id.toString(), user]));@@ -259,6 +273,6 @@ export class AdminSearchService {const user = usersById.get(userId.toString());if (user) orderedUsers.push(user);}- return Promise.all(orderedUsers.map((user) => mapUserToAdminResponse(user, cacheService, acls)));+ return Promise.all(orderedUsers.map((user) => mapUserToAdminResponse(user, cacheService, acls, locale)));}}diff --git a/fluxer_api/src/api/admin/tests/audit_coverage/UserAdminAuditCases.ts b/fluxer_api/src/api/admin/tests/audit_coverage/UserAdminAuditCases.tsindex 79b074b21..82ee91f22 100644--- a/fluxer_api/src/api/admin/tests/audit_coverage/UserAdminAuditCases.ts+++ b/fluxer_api/src/api/admin/tests/audit_coverage/UserAdminAuditCases.ts@@ -157,6 +157,29 @@ export const UserAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [};},},+ {+ method: 'GET',+ route: '/admin/users',+ name: 'acl',+ async prepare() {+ return {+ request: {path: `/admin/users?acl=${encodeURIComponent(AdminACLs.WILDCARD)}&limit=10&offset=0`},+ expected: {+ action: 'search_users',+ targetType: 'user',+ targetId: '0',+ metadata: {+ selector: 'acl',+ acl: AdminACLs.WILDCARD,+ limit: '10',+ offset: '0',+ result_count: '1',+ total: '1',+ },+ },+ };+ },+ },{method: 'GET',route: '/admin/users',diff --git a/fluxer_api/src/api/user/repositories/UserRepository.ts b/fluxer_api/src/api/user/repositories/UserRepository.tsindex d1f100760..a24a3f0b5 100644--- a/fluxer_api/src/api/user/repositories/UserRepository.ts+++ b/fluxer_api/src/api/user/repositories/UserRepository.ts@@ -155,6 +155,10 @@ export class UserRepository {return this.lookupRepo.findByStripeCustomerId(stripeCustomerId);}+ async listUserIdsWithAcl(acl: string): Promise<Array<UserID>> {+ return this.lookupRepo.listUserIdsWithAcl(acl);+ }+async listUserIdsByLastActiveIp(lastActiveIp: string,limit: number,@@ -192,6 +196,10 @@ export class UserRepository {return result ?? {last_active_at: null, last_active_ip: null};}+ async indexUserAcls(user: User): Promise<void> {+ return this.accountRepo.indexUserAcls(user);+ }+async deleteUserSecondaryIndices(userId: UserID): Promise<void> {return this.accountRepo.deleteUserSecondaryIndices(userId);}@@ -487,10 +495,6 @@ export class UserRepository {return this.channelRepo.listPrivateChannelSummaries(userId);}- async deleteAllPrivateChannels(userId: UserID): Promise<void> {- return this.channelRepo.deleteAllPrivateChannels(userId);- }-async findExistingDmState(user1Id: UserID, user2Id: UserID): Promise<Channel | null> {return this.channelRepo.findExistingDmState(user1Id, user2Id);}@@ -507,10 +511,6 @@ export class UserRepository {return this.channelRepo.isDmChannelOpen(userId, channelId);}- async openDmForUser(userId: UserID, channelId: ChannelID, isGroupDm?: boolean): Promise<void> {- return this.channelRepo.openDmForUser(userId, channelId, isGroupDm);- }-async openPrivateChannelForUser(userId: UserID, channel: Channel): Promise<void> {return this.channelRepo.openPrivateChannelForUser(userId, channel);}diff --git a/fluxer_api/src/api/user/repositories/account/UserAccountRepository.ts b/fluxer_api/src/api/user/repositories/account/UserAccountRepository.tsindex 4f0546c56..334a8e019 100644--- a/fluxer_api/src/api/user/repositories/account/UserAccountRepository.ts+++ b/fluxer_api/src/api/user/repositories/account/UserAccountRepository.ts@@ -219,6 +219,10 @@ export class UserAccountRepository {}}+ async indexUserAcls(user: User): Promise<void> {+ await this.indexRepo.indexAcls(user.id, user.acls);+ }+async deleteUserSecondaryIndices(userId: UserID): Promise<void> {const user = await this.findUnique(userId);if (!user) return;@@ -230,6 +234,7 @@ export class UserAccountRepository {user.stripeCustomerId,user.stripeSubscriptionId,user.lastActiveIp,+ user.acls,);}diff --git a/fluxer_api/src/api/user/repositories/account/UserLookupRepository.ts b/fluxer_api/src/api/user/repositories/account/UserLookupRepository.tsindex 198c3c017..0c0f65af5 100644--- a/fluxer_api/src/api/user/repositories/account/UserLookupRepository.ts+++ b/fluxer_api/src/api/user/repositories/account/UserLookupRepository.ts@@ -3,6 +3,7 @@import type {UserID} from '@app/api/BrandedTypes';import {fetchMany, fetchOne} from '@app/api/database/CassandraQueryExecution';import type {+ UserByAclRow,UserByLastActiveIpRow,UserByLastActiveIpTrustKeyRow,UserByStripeCustomerIdRow,@@ -11,6 +12,7 @@ import type {} from '@app/api/database/types/UserTypes';import type {User} from '@app/api/models/User';import {+ UserByAcl,UserByLastActiveIp,UserByLastActiveIpTrustKey,UserByStripeCustomerId,@@ -20,6 +22,10 @@ import {import type {UserEmailOwnershipRepository} from '@app/api/user/repositories/account/crud/UserEmailOwnershipRepository';import {getSameIpDecisionKey, normalizeIpString} from '@fluxer/ip_utils/src/IpAddress';+const FETCH_USER_IDS_BY_ACL_QUERY = UserByAcl.select({+ columns: ['user_id'],+ where: UserByAcl.where.eq('acl'),+});const FETCH_DISCRIMINATORS_BY_USERNAME_QUERY = UserByUsername.select({columns: ['discriminator', 'user_id'],where: UserByUsername.where.eq('username'),@@ -102,6 +108,11 @@ export class UserLookupRepository {return users.filter((user): user is User => user !== null);}+ async listUserIdsWithAcl(acl: string): Promise<Array<UserID>> {+ const rows = await fetchMany<Pick<UserByAclRow, 'user_id'>>(FETCH_USER_IDS_BY_ACL_QUERY.bind({acl}));+ return rows.map((row) => row.user_id);+ }+async listUserIdsByLastActiveIp(lastActiveIp: string,limit: number,diff --git a/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.ts b/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.tsindex 38b902a49..2a1071f44 100644--- a/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.ts+++ b/fluxer_api/src/api/user/repositories/account/crud/UserIndexRepository.ts@@ -4,6 +4,7 @@ import type {UserID} from '@app/api/BrandedTypes';import {BatchBuilder} from '@app/api/database/CassandraQueryExecution';import type {UserRow} from '@app/api/database/types/UserTypes';import {+ UserByAcl,UserByEmail,UserByLastActiveIp,UserByLastActiveIpTrustKey,@@ -16,6 +17,15 @@ import {getSameIpDecisionKey} from '@fluxer/ip_utils/src/IpAddress';export class UserIndexRepository {async syncIndices(data: UserRow, oldData?: UserRow | null): Promise<void> {const batch = new BatchBuilder();+ const acls = new Set(data.acls ?? []);+ for (const acl of acls) {+ batch.addPrepared(UserByAcl.upsertAll({acl, user_id: data.user_id}));+ }+ for (const acl of oldData?.acls ?? []) {+ if (!acls.has(acl)) {+ batch.addPrepared(UserByAcl.deleteByPk({acl, user_id: data.user_id}));+ }+ }if (data.username && data.discriminator != null && data.discriminator !== undefined) {batch.addPrepared(UserByUsername.upsertAll({@@ -122,6 +132,14 @@ export class UserIndexRepository {await batch.execute();}+ async indexAcls(userId: UserID, acls: ReadonlySet<string>): Promise<void> {+ const batch = new BatchBuilder();+ for (const acl of acls) {+ batch.addPrepared(UserByAcl.upsertAll({acl, user_id: userId}));+ }+ await batch.execute();+ }+async updateLastActiveIpIndex(userId: UserID,lastActiveIp: string,@@ -174,8 +192,12 @@ export class UserIndexRepository {stripeCustomerId?: string | null,stripeSubscriptionId?: string | null,lastActiveIp?: string | null,+ acls?: ReadonlySet<string> | null,): Promise<void> {const batch = new BatchBuilder();+ for (const acl of acls ?? []) {+ batch.addPrepared(UserByAcl.deleteByPk({acl, user_id: userId}));+ }batch.addPrepared(UserByUsername.deleteByPk({username: username.toLowerCase(),diff --git a/fluxer_api/src/api/worker/tasks/BackfillUserAclIndex.ts b/fluxer_api/src/api/worker/tasks/BackfillUserAclIndex.tsnew file mode 100644index 000000000..bc715d958--- /dev/null+++ b/fluxer_api/src/api/worker/tasks/BackfillUserAclIndex.ts@@ -0,0 +1,59 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {Config} from '@app/api/Config';+import {Logger} from '@app/api/Logger';+import type {UserRepository} from '@app/api/user/repositories/UserRepository';+import {getWorkerDependencies} from '@app/api/worker/WorkerContext';+import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';+import type {WorkerService} from '@app/api/worker/WorkerService';+import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';+import {ms} from 'itty-time';++const USER_ACL_INDEX_BACKFILL_DONE_KEY = 'backfill:user_acl_index:done';+const CLAIM_KEY = 'backfill:user_acl_index:queued';+const CLAIM_TTL_SECONDS = ms('6 hours') / 1000;+const DEFAULT_PAGE_SIZE = 500;++async function backfillUserAclIndex(+ userRepository: Pick<UserRepository, 'scanAllUsersPage' | 'indexUserAcls'>,+ pageSize = DEFAULT_PAGE_SIZE,+): Promise<number> {+ let indexed = 0;+ let pageState: string | null = null;+ do {+ const page = await userRepository.scanAllUsersPage(pageSize, pageState);+ for (const user of page.users) {+ if (user.acls.size === 0) continue;+ await userRepository.indexUserAcls(user);+ indexed++;+ }+ pageState = page.pageState;+ } while (pageState !== null);+ Logger.info({indexed}, 'Backfilled the user ACL index');+ return indexed;+}++export async function queueUserAclIndexBackfill(+ kvClient: Pick<IKVProvider, 'get' | 'setnx' | 'del'>,+ workerService: Pick<WorkerService, 'addJob'>,+): Promise<void> {+ if (!Config.instance.selfHosted) return;+ if ((await kvClient.get(USER_ACL_INDEX_BACKFILL_DONE_KEY)) !== null) return;+ if (!(await kvClient.setnx(CLAIM_KEY, '1', CLAIM_TTL_SECONDS))) return;+ try {+ await workerService.addJob('backfillUserAclIndex', {});+ } catch (error) {+ await kvClient.del(CLAIM_KEY);+ if (!(error instanceof WorkerQueueOverflowError)) throw error;+ Logger.warn('Dropped the user ACL index backfill job, jobs stream is at its limit');+ }+}++const backfillUserAclIndexTask: WorkerTaskHandler = async () => {+ const {userRepository, kvClient} = getWorkerDependencies();+ await backfillUserAclIndex(userRepository);+ await kvClient.set(USER_ACL_INDEX_BACKFILL_DONE_KEY, new Date().toISOString());+};++export default backfillUserAclIndexTask;