Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
10.2 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest';
import { appTile, batchedTemplate, batchedTile, createTileBatcher, readBatched, readFrames } from './tileBatch.js';

/** One tile of a batch answer, as api/tile_batch.py writes it. */
function frame(index, status, text) {
  const body = new TextEncoder().encode(text);
  const out = new Uint8Array(8 + body.length);
  const view = new DataView(out.buffer);
  view.setUint16(0, index, true);
  view.setUint16(2, status, true);
  view.setUint32(4, body.length, true);
  out.set(body, 8);
  return out;
}

const text = (data) => new TextDecoder().decode(data);
const settled = () => new Promise((resolve) => setTimeout(resolve, 0));

/**
 * The app's batch route: each request is held until the test writes its tiles,
 * in whatever order and chunks it likes.
 */
function fakeApp() {
  const requests = [];
  const fetch = vi.fn((url, { signal } = {}) => {
    const request = { url, signal, aborted: false };
    requests.push(request);
    return new Promise((resolve, reject) => {
      signal?.addEventListener('abort', () => {
        request.aborted = true;
        const error = new Error('aborted');
        error.name = 'AbortError';
        reject(error);
        request.controller?.error(error);
      });
      request.refuse = (status) => resolve({ ok: false, status, body: null, blob: async () => new Blob(['no']) });
      const body = new ReadableStream({ start: (controller) => (request.controller = controller) });
      request.answer = () => resolve({ ok: true, status: 200, body });
      request.write = (...chunks) => chunks.forEach((chunk) => request.controller.enqueue(chunk));
      request.end = () => request.controller.close();
    });
  });
  return { fetch, requests, keys: (request) => new URL(request.url, 'http://app').searchParams.get('t').split(',') };
}


describe('reading a batch answer', () => {
  it('hands over each tile once all of it is in, across any chunking', async () => {
    const whole = new Uint8Array([...frame(1, 200, 'second'), ...frame(0, 404, 'none here')]);
    // cut through a head and through a body
    const chunks = [whole.slice(0, 5), whole.slice(5, 11), whole.slice(11)];
    const body = new ReadableStream({
      start(controller) {
        chunks.forEach((chunk) => controller.enqueue(chunk));
        controller.close();
      },
    });
    const tiles = [];
    await readFrames(body, (index, status, data) => tiles.push([index, status, text(data)]));
    expect(tiles).toEqual([
      [1, 200, 'second'],
      [0, 404, 'none here'],
    ]);
  });
});

describe('asking tiles many at a time', () => {
  it('sends the asks of one moment as one request per source, the same tile once', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0 });
    const a = batcher.get('/api/tiles/batch/esri', 5, 1, 1);
    const b = batcher.get('/api/tiles/batch/esri', 5, 2, 1);
    const again = batcher.get('/api/tiles/batch/esri', 5, 1, 1);
    const relief = batcher.get('/api/terrain/batch', 5, 1, 1);
    await settled();
    expect(app.requests.map((request) => request.url.split('?')[0])).toEqual([
      '/api/tiles/batch/esri',
      '/api/terrain/batch',
    ]);
    expect(app.keys(app.requests[0])).toEqual(['5/1/1', '5/2/1']);
    const [imagery, terrain] = app.requests;
    imagery.answer();
    terrain.answer();
    // each tile is handed over as it arrives, in whatever order the app finishes
    imagery.write(frame(1, 200, 'B'));
    expect(text((await b).data)).toBe('B');
    imagery.write(frame(0, 200, 'A'));
    imagery.end();
    expect(text((await a).data)).toBe('A');
    expect(text((await again).data)).toBe('A');
    terrain.write(frame(0, 200, 'dem'));
    terrain.end();
    expect(await relief).toEqual({ status: 200, data: expect.any(ArrayBuffer) });
  });

  it('carries a failed tile’s status and reason, and fails a tile the app left out', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0 });
    const gap = batcher.get('/api/tiles/batch/esri', 5, 1, 1);
    const left = batcher.get('/api/tiles/batch/esri', 5, 2, 1);
    await settled();
    app.requests[0].answer();
    app.requests[0].write(frame(0, 404, 'no imagery'));
    app.requests[0].end();
    const answer = await gap;
    expect(answer.status).toBe(404);
    expect(text(answer.data)).toBe('no imagery');
    await expect(left).rejects.toThrow(/left a tile out/);
  });

  it('says when the app refused the request as a whole, so a tile can be asked alone', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0 });
    const tile = batcher.get('/api/tiles/batch/esri', 5, 1, 1);
    await settled();
    app.requests[0].refuse(404);
    await expect(tile).rejects.toMatchObject({ status: 404 });
  });

  it('keeps to its share of the connections and its size, and sends the rest as they free up', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0, max: 2, inFlight: 1 });
    const tiles = [0, 1, 2].map((x) => batcher.get('/api/tiles/batch/esri', 5, x, 0));
    await settled();
    expect(app.requests).toHaveLength(1);
    expect(app.keys(app.requests[0])).toEqual(['5/0/0', '5/1/0']);
    expect(batcher.flying).toBe(1);
    app.requests[0].answer();
    app.requests[0].write(frame(0, 200, 'a'), frame(1, 200, 'b'));
    app.requests[0].end();
    await Promise.all(tiles.slice(0, 2));
    await settled();
    expect(app.requests).toHaveLength(2);
    expect(app.keys(app.requests[1])).toEqual(['5/2/0']);
  });

  it('drops a tile no longer wanted before it goes, and stops a request nobody waits for', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0 });
    const early = new AbortController();
    const dropped = batcher.get('/api/tiles/batch/esri', 5, 1, 1, early.signal);
    const kept = new AbortController();
    const flying = batcher.get('/api/tiles/batch/esri', 5, 2, 1, kept.signal);
    early.abort();
    await expect(dropped).rejects.toMatchObject({ name: 'AbortError' });
    await settled();
    expect(app.keys(app.requests[0])).toEqual(['5/2/1']);
    kept.abort();
    await expect(flying).rejects.toMatchObject({ name: 'AbortError' });
    expect(app.requests[0].aborted).toBe(true);
    const gone = new AbortController();
    gone.abort();
    await expect(batcher.get('/api/tiles/batch/esri', 5, 3, 1, gone.signal)).rejects.toMatchObject({
      name: 'AbortError',
    });
  });

  it('keeps a request going while one of its tiles is still wanted', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0 });
    const gone = new AbortController();
    const dropped = batcher.get('/api/tiles/batch/esri', 5, 1, 1, gone.signal);
    const wanted = batcher.get('/api/tiles/batch/esri', 5, 2, 1);
    await settled();
    gone.abort();
    await expect(dropped).rejects.toMatchObject({ name: 'AbortError' });
    expect(app.requests[0].aborted).toBe(false);
    app.requests[0].answer();
    app.requests[0].write(frame(0, 200, 'late'), frame(1, 200, 'ok'));
    app.requests[0].end();
    expect(text((await wanted).data)).toBe('ok');
  });
});

describe('a tile through the app', () => {
  const asked = { providerId: 'esri-world-imagery', z: 5, x: 3, y: 4 };
  const single = (status, body) =>
    vi.fn(async () => ({
      ok: status === 200,
      status,
      statusText: '',
      blob: async () => new Blob([body]),
      arrayBuffer: async () => new TextEncoder().encode(body).buffer,
    }));

  it('asks the tile at the address it always had', async () => {
    const fetch = single(200, 'tile');
    const { data } = await appTile(asked, { fetch });
    expect(text(data)).toBe('tile');
    expect(fetch).toHaveBeenCalledWith('/api/tiles/esri-world-imagery/5/3/4', { signal: undefined });
  });

  it('hands over a tile that failed with its status and reason', async () => {
    const failure = await appTile(asked, { fetch: single(429, 'paused') }).catch((e) => e);
    expect(failure.status).toBe(429);
    expect(await failure.body.text()).toBe('paused');
  });
});

describe('a flat map’s free imagery, asked in batches', () => {
  it('names a tile at an address of its own, and reads it back', () => {
    expect(batchedTemplate('esri-world-imagery')).toBe('azimut-tiles://esri-world-imagery/{z}/{x}/{y}');
    expect(readBatched('azimut-tiles://esri-wayback~20/14/8712/6153')).toEqual({ providerId: 'esri-wayback~20', z: 14, x: 8712, y: 6153 });
    expect(readBatched('/api/tiles/esri-world-imagery/1/2/3')).toBeNull();
    expect(readBatched('azimut-billed://m1/sentinel2/1/2/3')).toBeNull();
  });

  it('hands the engine the tile, or the failure its own address would have given', async () => {
    const answers = { '5/1/1': { status: 200, data: new TextEncoder().encode('png').buffer }, '5/2/1': { status: 404, data: new TextEncoder().encode('no imagery at this location/zoom').buffer } };
    const batcher = { get: vi.fn(async (path, z, x, y) => answers[`${z}/${x}/${y}`]) };
    const signal = new AbortController().signal;
    const tile = await batchedTile({ providerId: 'esri-world-imagery', z: 5, x: 1, y: 1 }, { batcher, signal });
    expect(text(tile.data)).toBe('png');
    expect(batcher.get).toHaveBeenCalledWith('/api/tiles/batch/esri-world-imagery', 5, 1, 1, signal);
    const failure = await batchedTile({ providerId: 'esri-world-imagery', z: 5, x: 2, y: 1 }, { batcher }).catch((error) => error);
    expect(failure.status).toBe(404);
    expect(await failure.body.text()).toBe('no imagery at this location/zoom');
  });

  it('shares one request between the tiles asked in the same moment', async () => {
    const app = fakeApp();
    const batcher = createTileBatcher({ fetch: app.fetch, wait: 0 });
    const tiles = [batchedTile({ providerId: 'esri', z: 5, x: 1, y: 1 }, { batcher }), batchedTile({ providerId: 'esri', z: 5, x: 2, y: 1 }, { batcher })];
    await settled();
    expect(app.requests).toHaveLength(1);
    const [request] = app.requests;
    expect(app.keys(request)).toEqual(['5/1/1', '5/2/1']);
    request.answer();
    request.write(frame(1, 200, 'b'), frame(0, 200, 'a'));
    request.end();
    expect((await Promise.all(tiles)).map((tile) => text(tile.data))).toEqual(['a', 'b']);
  });
});