Skip to content

A local-first personal finance app

21.6 KB
Raw Blame
// @ts-strict-ignore
import * as connection from '@actual-app/core/platform/client/connection';
import { subDays } from '@actual-app/core/shared/months';
import { q } from '@actual-app/core/shared/query';
import { resetTracer, tracer } from '@actual-app/core/shared/test-helpers';

import { pagedQuery } from './pagedQuery';

function wait(n) {
  return new Promise(resolve => setTimeout(() => resolve(`wait(${n})`), n));
}

function isCountQuery(query) {
  if (query.selectExpressions.length === 1) {
    const select = query.selectExpressions[0];
    return select.result && select.result.$count === '*';
  }

  return false;
}

function select(row, selectExpressions) {
  return Object.fromEntries(
    selectExpressions.map(fieldName => [fieldName, row[fieldName]]),
  );
}

function selectData(data, selectExpressions) {
  return data.map(row => select(row, selectExpressions));
}

function limitOffset(data, limit, offset) {
  const start = offset != null ? offset : 0;
  const end = limit != null ? limit : data.length;
  return data.slice(start, start + end);
}

function runPagedQuery(query, data) {
  if (isCountQuery(query)) {
    return data.length;
  }

  if (query.filterExpressions.length > 0) {
    const filter = query.filterExpressions[0];
    if (filter.id != null) {
      return data.filter(row => row.id === filter.id);
    }

    if (filter.date != null) {
      const op = Object.keys(filter.date)[0];

      return limitOffset(
        data
          .filter(row => {
            return op === '$gte'
              ? row.date >= filter.date[op]
              : op === '$lte'
                ? row.date <= filter.date[op]
                : op === '$lt'
                  ? row.date < filter.date[op]
                  : op === '$gt'
                    ? row.date > filter.date[op]
                    : false;
          })
          .map(row => select(row, query.selectExpressions)),
        query.limit,
        query.offset,
      );
    }
  } else if (query.offset != null || query.limit != null) {
    return limitOffset(
      data.map(row => select(row, query.selectExpressions)),
      query.limit,
      query.offset,
    );
  }

  throw new Error('Unable to execute query: ' + JSON.stringify(query, null, 2));
}

const eventListeners = new Map();

function clearEventListeners() {
  eventListeners.clear();
}

function mockListen(name, listener): () => void {
  if (!eventListeners.get(name)) {
    eventListeners.set(name, []);
  }
  eventListeners.get(name).push(listener);

  return () => {
    const arr = eventListeners.get(name);
    eventListeners.set(
      name,
      arr.filter(l => l !== listener),
    );
  };
}

function mockPublishEvent(name, args) {
  const listeners = eventListeners.get(name);
  if (listeners) {
    listeners.forEach(listener => {
      listener(args);
    });
  }
}

async function mockSend(name, args, { delay }) {
  switch (name) {
    case 'query':
      const query = args;
      if (!isCountQuery(query)) {
        tracer.event('server-query');
      }
      if (delay) {
        await wait(delay);
      }
      return { data: query.selectExpressions, dependencies: ['transactions'] };
    default:
      throw new Error(`Command not implemented: ${name}`);
  }
}

function mockServer({ send = mockSend, listen = mockListen }) {
  vi.spyOn(connection, 'send').mockImplementation((name, args) => {
    return send(name, args, { delay: 0 });
  });
  vi.spyOn(connection, 'listen').mockImplementation(listen);
}

function clearMockServer() {
  clearEventListeners();
  vi.clearAllMocks();
}

function mockBasicServer(delay?) {
  mockServer({
    send: (name, args) => {
      return mockSend(name, args, { delay });
    },
  });
}

function mockPagingServer(
  dataLength,
  { delay, eventType = 'select' }: { delay?: number; eventType?: string } = {},
) {
  const data = [];
  for (let i = 0; i < dataLength; i++) {
    data.push({ id: i, date: subDays('2020-05-01', Math.floor(i / 5)) });
  }

  mockServer({
    send: async (name, args) => {
      switch (name) {
        case 'query':
          const query = args;
          tracer.event(
            'server-query',
            eventType === 'select' ? query.selectExpressions : query,
          );
          if (delay) {
            await wait(delay);
          }
          return {
            data: runPagedQuery(query, data),
            dependencies: ['transactions'],
          };
        default:
          throw new Error(`Command not implemented: ${name}`);
      }
    },
  });

  return data;
}

describe('pagedQuery', () => {
  beforeEach(() => {
    resetTracer();
    clearMockServer();
  });

  it(`runs and subscribes to a query`, async () => {
    mockBasicServer();
    tracer.start();

    const query = q('transactions').select('*');
    pagedQuery(query, { onData: data => tracer.event('data', data) });

    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);

    // Simulate a sync event
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });

    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);
  });

  it(`runs but ignores applied events (onlySync: true)`, async () => {
    mockBasicServer();
    tracer.start();

    const query = q('transactions').select('*');
    pagedQuery(query, {
      onData: data => tracer.event('data', data),
      options: { onlySync: true },
    });

    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);
    // Simulate a sync event
    mockPublishEvent('sync-event', {
      type: 'applied',
      tables: ['transactions'],
    });

    const p = Promise.race([tracer.wait('server-query'), wait(100)]);
    expect(await p).toEqual('wait(100)');
  });

  it(`runs and updates with sync events (onlySync: true)`, async () => {
    mockBasicServer();
    tracer.start();

    const query = q('transactions').select('*');
    pagedQuery(query, {
      onData: data => tracer.event('data', data),
      options: { onlySync: true },
    });

    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);
    // Simulate a sync event
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });
    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);
  });

  it(`cancels existing requests`, async () => {
    let requestId = 0;
    mockServer({
      send: async (name, args) => {
        switch (name) {
          case 'query':
            const query = args;
            if (!isCountQuery(query)) {
              requestId++;
            }
            await wait(500);
            return { data: requestId, dependencies: ['transactions'] };
          default:
            throw new Error(`Command not implemented: ${name}`);
        }
      },
    });

    tracer.start();
    const query = q('transactions').select('*');
    const lq = pagedQuery(query, {
      onData: data => tracer.event('data', data),
      options: { onlySync: true },
    });

    // Users should never call `run` manually but we'll do it to
    // test

    void lq.run();
    await wait(0);
    void lq.run();
    await wait(0);
    void lq.run();
    await wait(0);
    void lq.run();
    await wait(0);
    void lq.run();

    // Wait for the same delay the server has
    await wait(500);
    // Data should only be returned once
    await tracer.expect('data', 6);
  });

  it(`cancels requests when server pushes`, async () => {
    mockBasicServer();
    tracer.start();

    const query = q('transactions').select('*');

    pagedQuery(query, {
      onData: data => tracer.event('data', data),
      options: { onlySync: true },
    });

    // Simulate a sync event
    // Send a push in the middle of the query running for the first run
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });
    // The first request should get handled, but there should be no
    // `data` event
    await tracer.expect('server-query');

    // The live query simply reruns the query, ignoring the first result
    await tracer.expect('server-query');

    // And we have data!
    await tracer.expect('data', ['*']);
  });

  it(`reruns if data changes in the middle of *any* request`, async () => {
    mockBasicServer(500);
    tracer.start();

    const query = q('transactions').select('*');

    pagedQuery(query, {
      onData: data => tracer.event('data', data),
      options: { onlySync: true },
    });

    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);

    // Simulate a sync event
    // Send two pushes in a row
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });

    // Two requests will be made to the server, but the first one
    // should be ignored and we only get one data back
    await tracer.expect('server-query');
    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);
  });

  it(`unsubscribes correctly`, () => async done => {
    mockBasicServer();
    tracer.start();

    const query = q('transactions').select('*');

    const lq = pagedQuery(query, {
      onData: data => tracer.event('data', data),
    });

    await tracer.expect('server-query');
    await tracer.expect('data', ['*']);
    lq.unsubscribe();

    // Simulate a sync event
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });

    // Wait a bit and make sure nothing comes through
    const p = Promise.race([tracer.expect('server-query'), wait(100)]);
    await expect(p).resolves.toEqual('wait(100)');
    done();
  });

  it('pagedQuery makes requests in pages', () => async done => {
    const data = mockPagingServer(1502);
    tracer.start();

    const query = q('transactions').select('id');
    const paged = pagedQuery(query, {
      onData: data => tracer.event('data', data),
      onPageData: data => tracer.event('page-data', data),
    });

    await tracer.expect('server-query', [{ result: { $count: '*' } }]);
    await tracer.expect('server-query', ['id']);

    await tracer.expect('data', async d => {
      expect(d.length).toBe(500);
      expect(d[0].id).toBe(data[0].id);
    });

    expect(paged.totalCount).toBe(data.length);

    await paged.fetchNext();
    tracer.expectNow('server-query', ['id']);
    tracer.expectNow('page-data', d => {
      expect(d.length).toBe(500);
      expect(d[0].id).toBe(data[500].id);
    });
    tracer.expectNow('data', d => {
      expect(d.length).toBe(1000);
    });
    expect(paged.hasNext).toBe(true);

    await paged.fetchNext();
    tracer.expectNow('server-query', ['id']);
    tracer.expectNow('page-data', d => {
      expect(d.length).toBe(500);
      expect(d[0].id).toBe(data[1000].id);
    });
    tracer.expectNow('data', d => {
      expect(d.length).toBe(1500);
    });
    expect(paged.hasNext).toBe(true);

    await paged.fetchNext();
    tracer.expectNow('server-query', ['id']);
    tracer.expectNow('page-data', d => {
      expect(d.length).toBe(2);
      expect(d[0].id).toBe(data[1500].id);
    });
    tracer.expectNow('data', d => {
      expect(d.length).toBe(1502);
    });

    expect(paged.data).toEqual(selectData(data, ['id']));
    expect(paged.hasNext).toBe(false);

    await paged.fetchNext();
    // Wait a bit and make sure nothing comes through
    const p = Promise.race([tracer.expect('server-query'), wait(100)]);
    expect(await p).toEqual('wait(100)');
    done();
  });

  it('pagedQuery allows customizing page count', async () => {
    const data = mockPagingServer(50);
    tracer.start();

    const query = q('transactions').select('id');
    pagedQuery(query, {
      onData: data => tracer.event('data', data),
      options: { pageCount: 10 },
    });

    await tracer.expect('server-query', [{ result: { $count: '*' } }]);
    await tracer.expect('server-query', ['id']);

    // Should only get 10 items back
    await tracer.expect('data', selectData(data, ['id']).slice(0, 10));
  });

  it('pagedQuery only runs `fetchNext` once at a time', () => async done => {
    mockPagingServer(1000, { delay: 200 });
    tracer.start();

    const query = q('transactions').select('id');
    const paged = pagedQuery(query, {
      onData: data => tracer.event('data', data),
    });

    await tracer.expect('server-query', [{ result: { $count: '*' } }]);
    await tracer.expect('server-query', ['id']);
    await tracer.expect('data', vi.fn());

    void paged.fetchNext();
    void paged.fetchNext();
    await wait(2);
    void paged.fetchNext();

    await tracer.expect('server-query', ['id']);
    await tracer.expect('data', vi.fn());

    // Wait a bit and make sure nothing comes through
    const p = Promise.race([tracer.expect('server-query'), wait(200)]);
    expect(await p).toEqual('wait(200)');
    done();
  });

  it('pagedQuery refetches all paged data on update', async () => {
    const data = mockPagingServer(500, { delay: 200 });
    tracer.start();

    const query = q('transactions').select('id');
    const paged = pagedQuery(query, {
      onData: data => tracer.event('data', data),
      onPageData: data => tracer.event('page-data', data),
      options: { pageCount: 20 },
    });

    await tracer.expect('server-query', [{ result: { $count: '*' } }]);
    await tracer.expect('server-query', ['id']);
    await tracer.expect('data', d => {
      expect(d.length).toBe(20);
    });

    await paged.fetchNext();

    await tracer.expect('server-query', ['id']);
    await tracer.expect('page-data', d => {
      expect(d.length).toBe(20);
      expect(d[0].id).toBe(data[20].id);
    });
    await tracer.expect('data', d => {
      expect(d.length).toBe(40);
    });

    // Simulate a sync event
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });

    await tracer.expect('server-query', [{ result: { $count: '*' } }]);
    await tracer.expect('server-query', ['id']);
    await tracer.expect('data', d => {
      // All 40 we fetched again
      expect(d.length).toBe(40);
    });
  });

  it('pagedQuery reruns `fetchNext` if data changed underneath it', async () => {
    const data = mockPagingServer(500, { delay: 10 });
    const query = q('transactions').select('id');
    const paged = pagedQuery(query, {
      onData: data => tracer.event('data', data),
      onPageData: data => tracer.event('page-data', data),
      options: { pageCount: 20 },
    });

    await paged.fetchNext();

    tracer.start();

    void paged.fetchNext().then(() => {
      tracer.event('page-finished');
    });

    await wait(1);
    // Simulate a sync event
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });

    // This is from the paged request, but it ignores the new data
    await tracer.expect('server-query', ['id']);

    await tracer.expect('server-query', [{ result: { $count: '*' } }]);
    await tracer.expect('server-query', ['id']);
    await tracer.expect('data', d => {
      expect(d.length).toBe(40);
    });

    // Now the paged request reruns
    await tracer.expect('server-query', ['id']);
    await tracer.expect('page-data', d => {
      expect(d.length).toBe(20);
      expect(d[0].id).toBe(data[40].id);
    });
    await tracer.expect('data', d => {
      expect(d.length).toBe(60);
    });

    // Make sure the page promise is never resolved until everything
    // has settled
    await tracer.expect('page-finished');
  });

  it('pagedQuery fetches up to a specific row', async () => {
    const data = mockPagingServer(500, { delay: 10, eventType: 'all' });
    const query = q('transactions').select(['id', 'date']);
    const paged = pagedQuery(query, {
      onData: data => tracer.event('data', data),
      onPageData: data => tracer.event('page-data', data),
      options: { pageCount: 20 },
    });
    await paged.run();

    tracer.start();

    const item = data.find(row => row.id === 300);
    void paged.refetchUpToRow(item.id, { field: 'date', order: 'desc' });

    await tracer.expect(
      'server-query',
      expect.objectContaining({
        selectExpressions: [{ result: { $count: '*' } }],
      }),
    );
    await tracer.expect(
      'server-query',
      expect.objectContaining({ filterExpressions: [{ id: 300 }] }),
    );
    await tracer.expect(
      'server-query',
      expect.objectContaining({
        filterExpressions: [{ date: { $gte: item.date } }],
      }),
    );
    await tracer.expect(
      'server-query',
      expect.objectContaining({
        filterExpressions: [{ date: { $lt: item.date } }],
        limit: 20,
      }),
    );
    await tracer.expect(
      'data',
      data.slice(0, data.findIndex(row => row.date < item.date) + 20),
    );

    await wait(1000);
  });

  // Each query takes 50ms and fetching up to a row takes three, so a plain
  // refresh from an early sync would finish before it, and from a late one
  // after it
  it.each([
    ['early', 5],
    ['late', 120],
  ])(
    'pagedQuery fetches up to a row through a sync arriving %s',
    async (_, syncDelay) => {
      mockPagingServer(500, { delay: 50 });
      const query = q('transactions').select(['id', 'date']);
      const paged = pagedQuery<{ id: number }>(query, {
        options: { pageCount: 20 },
      });
      await paged.run();

      const refetch = paged.refetchUpToRow(300, {
        field: 'date',
        order: 'desc',
      });
      await wait(syncDelay);
      mockPublishEvent('sync-event', {
        type: 'success',
        tables: ['transactions'],
      });
      await refetch;

      expect(paged.data.some(row => row.id === 300)).toBe(true);
    },
  );

  it('pagedQuery drops a row deleted while fetching up to it', async () => {
    const data = [];
    for (let i = 0; i < 500; i++) {
      data.push({ id: i, date: subDays('2020-05-01', Math.floor(i / 5)) });
    }
    mockServer({
      send: async (name, query) => {
        const result = runPagedQuery(query, data);
        // A sync deletes it while the extra page after it loads
        if (query.filterExpressions[0]?.date?.$lt && data[300].id === 300) {
          data.splice(300, 1);
          mockPublishEvent('sync-event', {
            type: 'success',
            tables: ['transactions'],
          });
        }
        return { data: result, dependencies: ['transactions'] };
      },
    });
    const query = q('transactions').select(['id', 'date']);
    const paged = pagedQuery<{ id: number }>(query, {
      options: { pageCount: 20 },
    });
    await paged.run();

    await paged.refetchUpToRow(300, { field: 'date', order: 'desc' });

    expect(paged.data.some(row => row.id === 300)).toBe(false);
  });

  it('does not spin forever when unsubscribed during a refetch', async () => {
    // Regression test: unsubscribing while a refetch was in flight (e.g.
    // switching accounts while a scroll-triggered `fetchNext` was pending)
    // used to leave `inflightRequestId` set forever, making `fetchNext` spin
    // in an infinite microtask loop that froze the app at 100% CPU.
    const data = [];
    for (let i = 0; i < 400; i++) {
      data.push({ id: i });
    }

    // Server that blocks every page query until we release it, so the test
    // controls exactly when each in-flight request resolves.
    const pending = [];
    mockServer({
      send: async (name, args) => {
        switch (name) {
          case 'query': {
            const query = args;
            if (isCountQuery(query)) {
              return { data: data.length, dependencies: ['transactions'] };
            }
            await new Promise(resolve => pending.push(resolve));
            return {
              data: limitOffset(
                data.map(row => select(row, query.selectExpressions)),
                query.limit,
                query.offset,
              ),
              dependencies: ['transactions'],
            };
          }
          default:
            throw new Error(`Command not implemented: ${name}`);
        }
      },
    });

    const query = q('transactions').select('id');
    const paged = pagedQuery(query, {
      options: { onlySync: true, pageCount: 150 },
    });

    await wait(1);
    pending.shift()();
    await wait(1);
    expect(paged.data.length).toBe(150);

    // A scroll near the bottom starts fetching the next page...
    let fetchNextSettled = false;
    const fetchNextPromise = paged.fetchNext().then(() => {
      fetchNextSettled = true;
    });
    await wait(1);

    // ...a database change triggers a refetch while that page is loading...
    mockPublishEvent('sync-event', {
      type: 'success',
      tables: ['transactions'],
    });
    await wait(1);

    // ...and the account switch unsubscribes before either request finishes.
    paged.unsubscribe();
    while (pending.length) {
      pending.shift()();
    }

    // Pump the microtask queue a bounded number of times. Timers never fire
    // while `fetchNext` spins, so this must not wait on a timer.
    for (let i = 0; i < 2000; i++) {
      await Promise.resolve();
    }

    // The cancelled refetch must not leave a stale in-flight request id
    // behind — `fetchNext` loops on it until it clears.
    const internals = paged as unknown as {
      _inflightRequestId: number | null;
    };
    const staleRequestId = internals._inflightRequestId;
    // Break the loop if it is spinning, so a regression fails cleanly here
    // instead of hanging the test worker.
    internals._inflightRequestId = null;
    expect(staleRequestId).toBeNull();

    // fetchNext recovers: release the page request it retries and let it
    // finish.
    for (let i = 0; i < 20 && !fetchNextSettled; i++) {
      while (pending.length) {
        pending.shift()();
      }
      await wait(1);
    }
    await fetchNextPromise;
    expect(fetchNextSettled).toBe(true);
    expect(paged.data.length).toBe(300);
  });
});