Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

21.2 KB
Raw Blame
import { ForbiddenException, HttpException, HttpStatus } from '@nestjs/common'
import { generateKeyPairSync } from 'crypto'

jest.mock('../project/project.service', () => ({ ProjectService: class {} }))
jest.mock('../project/entity/project.entity', () => ({ Project: class {} }))
jest.mock('./bigquery.service', () => ({ BigQueryService: class {} }))
jest.mock('./data-export.crypto', () => ({
  encryptWarehouseCredentials: jest.fn((value) => `encrypted:${value}`),
  decryptWarehouseCredentials: jest.fn((value) => value.slice(10)),
}))

import { DataExportService } from './data-export.service'
import { DataExport } from './entity/data-export.entity'
import { SaveDataExportDto, validateExportDate } from './data-export.dto'
import { WarehouseExportError } from './data-export.schema'
import {
  PlanCode,
  PlanFeatureCode,
  PlanType,
  userHasPlanFeature,
} from '../user/entities/user.entity'

const key = generateKeyPairSync('rsa', { modulusLength: 2048 })
  .privateKey.export({ format: 'pem', type: 'pkcs8' })
  .toString()
const credentials = JSON.stringify({
  type: 'service_account',
  project_id: 'warehouse-project',
  client_email: '[email protected]',
  private_key: key,
})

const makeConnection = (overrides: Partial<DataExport> = {}): DataExport =>
  ({
    id: 'connection-id',
    revision: 'revision-id',
    ownerId: 'owner-id',
    projectId: 'abcdefghijkl',
    gcpProjectId: 'warehouse-project',
    datasetId: 'analytics',
    location: 'EU',
    tablePrefix: 'swetrix_abcdefghijkl',
    encryptedCredentials: `encrypted:${credentials}`,
    serviceAccountEmail: '[email protected]',
    startDate: '2026-10-01',
    nextDate: '2026-10-27',
    enabled: true,
    status: 'idle',
    nextSyncAt: new Date('2026-10-27T06:00:00Z'),
    lastSyncedDate: '2026-10-26',
    lastSyncedAt: new Date('2026-10-26T06:00:00Z'),
    lastRowCount: 100,
    lastError: null,
    createdAt: new Date('2026-10-01T00:00:00Z'),
    ...overrides,
  }) as DataExport

function setup(initial: DataExport | null = makeConnection()) {
  let record = initial
  const repository: any = {
    findOneBy: jest.fn(async (where) => {
      if (!record) return null
      if (where.id && where.id !== record.id) return null
      if (where.revision && where.revision !== record.revision) return null
      return { ...record }
    }),
    findOne: jest.fn(async () => record && { ...record }),
    find: jest.fn(async () => (record ? [{ ...record }] : [])),
    update: jest.fn(async (where, value) => {
      const matches =
        record &&
        (!where.id || where.id === record.id) &&
        (!where.revision || where.revision === record.revision) &&
        (where.enabled === undefined || where.enabled === record.enabled) &&
        (!where.status ||
          (typeof where.status === 'string'
            ? where.status === record.status
            : where.status.value.includes(record.status)))
      if (matches) {
        record = { ...record, ...value }
      }
      return { affected: matches ? 1 : 0 }
    }),
    save: jest.fn(async (value) => {
      record = { ...record, ...value }
      return record
    }),
    delete: jest.fn(async () => {
      record = null
    }),
    create: jest.fn((value) => value),
  }
  repository.createQueryBuilder = jest.fn(() => {
    const builder: any = {
      addSelect: jest.fn(() => builder),
      where: jest.fn(() => builder),
      getOne: jest.fn(async () => record && { ...record }),
    }
    return builder
  })
  repository.manager = {
    transaction: async (fn: any) => fn({ getRepository: () => repository }),
  }
  const project = {
    id: 'abcdefghijkl',
    admin: {
      id: 'owner-id',
      planCode: PlanCode['100k'],
      planType: PlanType.plus,
    },
  }
  const projects: any = {
    getFullProject: jest.fn(async () => project),
    allowedToManage: jest.fn(),
    projectOwnerHasFeature: jest.fn((project, feature) =>
      userHasPlanFeature(project?.admin, feature),
    ),
    assertProjectOwnerFeatureAccess: jest.fn((project, feature) => {
      if (!userHasPlanFeature(project?.admin, feature)) {
        throw new HttpException(
          'This feature is not available on the current project owner plan.',
          HttpStatus.PAYMENT_REQUIRED,
        )
      }
    }),
  }
  const bigQuery: any = {
    validate: jest.fn(async () => {}),
    exportDay: jest.fn(async () => 10),
  }
  const queue: any = {
    getJob: jest.fn(async () => null),
    add: jest.fn(async () => ({})),
  }
  const service = new DataExportService(repository, projects, bigQuery, queue)
  const job = {
    connectionId: 'connection-id',
    revision: 'revision-id',
    runId: 'run-id',
  }
  const dto = (extra = {}): SaveDataExportDto => ({
    gcpProjectId: 'warehouse-project',
    datasetId: 'analytics',
    location: 'EU',
    tablePrefix: 'swetrix_abcdefghijkl',
    startDate: '2026-10-01',
    enabled: true,
    ...extra,
  })
  return {
    service,
    repository,
    projects,
    project,
    bigQuery,
    queue,
    job,
    dto,
    record: () => record,
  }
}

describe('warehouse orchestration', () => {
  beforeEach(() => {
    jest.useFakeTimers().setSystemTime(new Date('2026-10-27T10:00:00Z'))
  })
  afterEach(() => jest.useRealTimers())

  it('never returns stored credentials or internal owner/revision data', async () => {
    const { service } = setup()
    const result = await service.get('abcdefghijkl', 'owner-id')
    expect(JSON.stringify(result)).not.toContain(key)
    expect(result.connection).not.toHaveProperty('encryptedCredentials')
    expect(result.connection).not.toHaveProperty('ownerId')
    expect(result.connection).not.toHaveProperty('revision')
  })

  it('rejects a viewer even for a public project', async () => {
    const { service, projects, bigQuery, dto } = setup()
    projects.allowedToManage.mockImplementation(() => {
      throw new ForbiddenException()
    })
    await expect(service.save('abcdefghijkl', 'viewer', dto())).rejects.toThrow(
      ForbiddenException,
    )
    await expect(
      service.authorizeExport('abcdefghijkl', 'viewer'),
    ).rejects.toThrow(ForbiddenException)
    expect(bigQuery.validate).not.toHaveBeenCalled()
  })

  it.each([
    [PlanCode['100k'], PlanType.plus, true],
    [PlanCode['100k'], PlanType.enterprise, true],
    [PlanCode['100k'], PlanType.standard, false],
    [PlanCode['100k'], null, false],
    [PlanCode.trial, PlanType.plus, false],
    [PlanCode.free, PlanType.plus, false],
    [PlanCode.none, PlanType.enterprise, false],
  ])(
    'uses the existing plan policy for %s / %s (warehouse access: %s)',
    async (planCode, planType, allowed) => {
      const { service, project, projects } = setup()
      Object.assign(project.admin, { planCode, planType })
      expect(
        userHasPlanFeature(project.admin, PlanFeatureCode.dataWarehouse),
      ).toBe(allowed)
      if (allowed) {
        await expect(
          service.authorizeExport(project.id, 'owner-id'),
        ).resolves.toBe(project)
      } else {
        await expect(
          service.authorizeExport(project.id, 'owner-id'),
        ).rejects.toMatchObject({ status: HttpStatus.PAYMENT_REQUIRED })
      }
      expect(projects.assertProjectOwnerFeatureAccess).toHaveBeenCalledWith(
        project,
        PlanFeatureCode.dataWarehouse,
      )
    },
  )

  it.each([
    'raw export',
    'test connection',
    'sync now',
    'create enabled',
    'create paused',
    'change destination while paused',
    'rotate credentials while paused',
    'resume',
    'save enabled',
  ])('rejects %s for a Standard owner before external work', async (action) => {
    const initial = action.startsWith('create')
      ? null
      : makeConnection({ enabled: action !== 'resume' })
    const { service, project, bigQuery, queue, repository, dto } =
      setup(initial)
    project.admin.planType = PlanType.standard
    let operation: Promise<unknown>
    switch (action) {
      case 'raw export':
        operation = service.authorizeExport(project.id, 'owner-id')
        break
      case 'test connection':
        operation = service.test(project.id, 'owner-id', dto())
        break
      case 'sync now':
        operation = service.sync(project.id, 'owner-id')
        break
      default:
        operation = service.save(
          project.id,
          'owner-id',
          dto({
            ...(action.startsWith('create') ? { credentials } : {}),
            ...(action === 'change destination while paused'
              ? { datasetId: 'another_dataset' }
              : {}),
            ...(action === 'rotate credentials while paused'
              ? { credentials }
              : {}),
            enabled: !action.includes('paused'),
          }),
        )
    }
    await expect(operation).rejects.toMatchObject({
      status: HttpStatus.PAYMENT_REQUIRED,
    })
    expect(bigQuery.validate).not.toHaveBeenCalled()
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(queue.add).not.toHaveBeenCalled()
    expect(repository.save).not.toHaveBeenCalled()
  })

  it('keeps status, unchanged pause and disconnect available after a downgrade', async () => {
    const { service, project, dto, bigQuery, queue, record, projects } = setup()
    project.admin.planType = PlanType.standard
    await expect(service.get(project.id, 'owner-id')).resolves.toHaveProperty(
      'connection.id',
      'connection-id',
    )
    await service.save(project.id, 'owner-id', dto({ enabled: false }))
    expect(record().enabled).toBe(false)
    await service.remove(project.id, 'owner-id')
    expect(record()).toBeNull()
    expect(projects.assertProjectOwnerFeatureAccess).not.toHaveBeenCalled()
    expect(bigQuery.validate).not.toHaveBeenCalled()
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(queue.add).not.toHaveBeenCalled()
  })

  it('uses the billed project owner plan rather than the requesting manager plan', async () => {
    const { service, project, projects } = setup()
    await expect(
      service.authorizeExport(project.id, 'standard-manager'),
    ).resolves.toBe(project)
    expect(projects.allowedToManage).toHaveBeenLastCalledWith(
      project,
      'standard-manager',
    )
    project.admin.planType = PlanType.standard
    await expect(
      service.authorizeExport(project.id, 'plus-manager'),
    ).rejects.toMatchObject({ status: HttpStatus.PAYMENT_REQUIRED })
    expect(projects.allowedToManage).toHaveBeenLastCalledWith(
      project,
      'plus-manager',
    )
    expect(projects.assertProjectOwnerFeatureAccess).toHaveBeenLastCalledWith(
      project,
      PlanFeatureCode.dataWarehouse,
    )
  })

  it('disables a scheduled export after downgrade without adding a queue job', async () => {
    const { service, project, queue, bigQuery, record } = setup()
    project.admin.planType = PlanType.standard
    await service.schedule()
    expect(queue.add).not.toHaveBeenCalled()
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(record()).toMatchObject({
      enabled: false,
      status: 'error',
      nextSyncAt: null,
      nextDate: '2026-10-27',
      lastSyncedDate: '2026-10-26',
    })
    expect(record().lastError).toContain('Plus')
  })

  it('disables an already queued export after downgrade without losing its checkpoint', async () => {
    const { service, project, bigQuery, job, record } = setup(
      makeConnection({ status: 'queued' }),
    )
    project.admin.planType = PlanType.standard
    await service.process(job)
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(record()).toMatchObject({
      enabled: false,
      status: 'error',
      nextSyncAt: null,
      nextDate: '2026-10-27',
      lastSyncedDate: '2026-10-26',
    })
    expect(record().lastError).toContain('Plus')
  })

  it('preserves the plan error when a queued job or retry runs after the scheduler disabled it', async () => {
    const { service, project, bigQuery, job, record } = setup(
      makeConnection({ status: 'queued' }),
    )
    project.admin.planType = PlanType.standard
    await service.schedule()
    await service.process(job)
    await service.process(job)
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(record()).toMatchObject({
      enabled: false,
      status: 'error',
      nextSyncAt: null,
    })
    expect(record().lastError).toContain('Plus')
  })

  it('preserves a scheduler downgrade during the final day instead of reporting success', async () => {
    const { service, project, bigQuery, job, record } = setup(
      makeConnection({ startDate: '2026-10-26', nextDate: '2026-10-26' }),
    )
    bigQuery.exportDay.mockImplementationOnce(async () => {
      project.admin.planType = PlanType.standard
      await service.schedule()
      return 10
    })
    await service.process(job)
    expect(bigQuery.exportDay).toHaveBeenCalledTimes(1)
    expect(record()).toMatchObject({
      enabled: false,
      status: 'error',
      nextSyncAt: null,
      nextDate: '2026-10-27',
      lastSyncedDate: '2026-10-26',
    })
    expect(record().lastError).toContain('Plus')
  })

  it('stops between days after downgrade and preserves the completed day checkpoint', async () => {
    const { service, project, bigQuery, job, record } = setup(
      makeConnection({ nextDate: '2026-10-01', lastSyncedDate: null }),
    )
    bigQuery.exportDay.mockImplementationOnce(async () => {
      project.admin.planType = PlanType.standard
      return 10
    })
    await service.process(job)
    expect(bigQuery.exportDay).toHaveBeenCalledTimes(1)
    expect(record()).toMatchObject({
      enabled: false,
      status: 'error',
      nextSyncAt: null,
      nextDate: '2026-10-02',
      lastSyncedDate: '2026-10-01',
      lastRowCount: 10,
    })
    expect(record().lastError).toContain('Plus')
  })

  it('refreshes the latest three completed days without moving the high-water mark backwards', async () => {
    const { service, bigQuery, job, record } = setup()
    await service.process(job)
    expect(bigQuery.exportDay.mock.calls.map((call) => call[2])).toEqual([
      '2026-10-24',
      '2026-10-25',
      '2026-10-26',
    ])
    expect(
      bigQuery.exportDay.mock.calls.every((call) => call[3] === 'run-id'),
    ).toBe(true)
    expect(record().nextDate).toBe('2026-10-27')
    expect(record().lastSyncedDate).toBe('2026-10-26')
    expect(record().lastRowCount).toBe(30)
    expect(record().nextSyncAt.toISOString()).toBe('2026-10-28T06:00:00.000Z')
  })

  it('refreshes the prior days even when there is a new daily partition to export', async () => {
    const { service, bigQuery, job } = setup(
      makeConnection({ nextDate: '2026-10-26' }),
    )
    await service.process(job)
    expect(bigQuery.exportDay.mock.calls.map((call) => call[2])).toEqual([
      '2026-10-24',
      '2026-10-25',
      '2026-10-26',
    ])
  })

  it('bounds a historical batch to seven days then leaves it due for continuation', async () => {
    const { service, bigQuery, job, record } = setup(
      makeConnection({ nextDate: '2026-10-01' }),
    )
    await service.process(job)
    expect(bigQuery.exportDay).toHaveBeenCalledTimes(7)
    expect(record().nextDate).toBe('2026-10-08')
    expect(record().nextSyncAt.getTime()).toBe(Date.now())
  })

  it('never exports before the selected start date or includes the current day', async () => {
    const { service, bigQuery, job } = setup(
      makeConnection({ startDate: '2026-10-26', nextDate: '2026-10-26' }),
    )
    await service.process(job)
    expect(bigQuery.exportDay).toHaveBeenCalledTimes(1)
    expect(bigQuery.exportDay.mock.calls[0][2]).toBe('2026-10-26')
  })

  it('does not advance a failed day and stores only safe provider errors', async () => {
    const { service, bigQuery, job, record } = setup(
      makeConnection({ nextDate: '2026-10-01' }),
    )
    bigQuery.exportDay
      .mockResolvedValueOnce(10)
      .mockRejectedValueOnce(new Error(`secret=${key}`))
    await expect(service.process(job)).rejects.toThrow('Export failed.')
    expect(record().nextDate).toBe('2026-10-02')
    expect(record().lastError).not.toContain(key)
    expect(record().nextSyncAt.toISOString()).toBe('2026-10-27T16:00:00.000Z')
  })

  it('preserves actionable sanitized provider failures', async () => {
    const { service, bigQuery, job, record } = setup()
    bigQuery.exportDay.mockRejectedValue(
      new WarehouseExportError(
        'The dataset location does not match the selected location.',
      ),
    )
    await expect(service.process(job)).rejects.toThrow('dataset location')
    expect(record().status).toBe('error')
  })

  it('stops a queued paused job and clears stale queued status', async () => {
    const { service, bigQuery, job, record } = setup(
      makeConnection({ enabled: false, status: 'queued' }),
    )
    await service.process(job)
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(record().status).toBe('idle')
  })

  it('stops before the next day after a pause during an export', async () => {
    const { service, bigQuery, job, repository, record } = setup()
    bigQuery.exportDay.mockImplementationOnce(async () => {
      await repository.update({}, { enabled: false })
      return 10
    })
    await service.process(job)
    expect(bigQuery.exportDay).toHaveBeenCalledTimes(1)
    expect(record().status).toBe('idle')
  })

  it('clears syncing status when manually paused during the final day', async () => {
    const { service, bigQuery, job, repository, record } = setup(
      makeConnection({ startDate: '2026-10-26', nextDate: '2026-10-26' }),
    )
    bigQuery.exportDay.mockImplementationOnce(async () => {
      await repository.update({}, { enabled: false })
      return 10
    })
    await service.process(job)
    expect(record().enabled).toBe(false)
    expect(record().status).toBe('idle')
    expect(record().nextDate).toBe('2026-10-27')
  })

  it('disables the connector after an ownership transfer before exporting data', async () => {
    const { service, bigQuery, job, project, record } = setup()
    project.admin.id = 'new-owner'
    await service.process(job)
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
    expect(record().enabled).toBe(false)
    expect(record().lastError).toContain('ownership changed')
  })

  it('rechecks ownership between days', async () => {
    const { service, bigQuery, job, project, record } = setup()
    bigQuery.exportDay.mockImplementationOnce(async () => {
      project.admin.id = 'new-owner'
      return 10
    })
    await service.process(job)
    expect(bigQuery.exportDay).toHaveBeenCalledTimes(1)
    expect(record().enabled).toBe(false)
  })

  it('will not reuse the previous owner credentials', async () => {
    const { service, project, dto } = setup()
    project.admin.id = 'new-owner'
    await expect(
      service.test('abcdefghijkl', 'new-owner', dto()),
    ).rejects.toThrow('Provide a Google service account')
  })

  it('allows pause while active but rejects destination changes', async () => {
    const { service, queue, dto, record } = setup(
      makeConnection({ status: 'syncing' }),
    )
    queue.getJob.mockResolvedValue({ getState: async () => 'active' })
    await service.save('abcdefghijkl', 'owner-id', dto({ enabled: false }))
    expect(record().enabled).toBe(false)
    await expect(
      service.save('abcdefghijkl', 'owner-id', dto({ datasetId: 'different' })),
    ).rejects.toThrow('Wait for the current export')
  })

  it('can pause without decrypting an unreadable stored key', async () => {
    const { service, dto, record } = setup(
      makeConnection({ encryptedCredentials: 'corrupted-key' }),
    )
    await service.save('abcdefghijkl', 'owner-id', dto({ enabled: false }))
    expect(record().enabled).toBe(false)
  })

  it('rotates credentials without re-exporting all historical data', async () => {
    const { service, dto, record } = setup()
    await service.save('abcdefghijkl', 'owner-id', dto({ credentials }))
    expect(record().nextDate).toBe('2026-10-27')
    expect(record().lastSyncedDate).toBe('2026-10-26')
    expect(record().revision).not.toBe('revision-id')
  })

  it('resets history when moving to a new destination', async () => {
    const { service, dto, record } = setup()
    await service.save(
      'abcdefghijkl',
      'owner-id',
      dto({ datasetId: 'new_dataset' }),
    )
    expect(record().nextDate).toBe('2026-10-01')
    expect(record().lastSyncedDate).toBeNull()
  })

  it('persists a configured destination for scheduler recovery when Redis is down', async () => {
    const { service, dto, queue, record } = setup(null)
    queue.add.mockRejectedValue(new Error('Redis disconnected'))
    await service.save('abcdefghijkl', 'owner-id', dto({ credentials }))
    expect(record().enabled).toBe(true)
    expect(record().status).toBe('error')
    expect(record().nextSyncAt.getTime()).toBe(Date.now())
  })

  it('disconnects an active job without deleting warehouse data or retaining credentials', async () => {
    const { service, queue, bigQuery, record } = setup()
    const remove = jest.fn()
    queue.getJob.mockResolvedValue({ getState: async () => 'active', remove })
    await service.remove('abcdefghijkl', 'owner-id')
    expect(record()).toBeNull()
    expect(remove).not.toHaveBeenCalled()
    expect(bigQuery.exportDay).not.toHaveBeenCalled()
  })

  it.each(['2026-02-30', '2026-10-27', '2026-10-28', '1999-12-31', 'invalid'])(
    'rejects invalid or incomplete UTC date %s',
    (value) => {
      expect(() => validateExportDate(value)).toThrow()
    },
  )
})