Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
9.4 KB
Raw Blame
/**
 * Tiles asked of the app many at a time: the globe's pictures and heights
 * (`lib/globe`, the 3D view).
 *
 * Every tile comes through the app, over the six connections a browser keeps to
 * one host. Tilted toward the horizon, a view asks for a hundred tiles at once
 * and each one used to wait for a free connection before the app even heard of
 * it. So the tiles asked in one go are sent as one request per source
 * (api/tile_batch.py), at most `BATCH_IN_FLIGHT` at a time so the app's other
 * calls keep a connection. The app fetches them side by side and writes each one
 * as it is done, and each is handed over as it arrives: a slow tile holds up
 * none of the others.
 *
 * The flat map asks a free provider the app keeps on disk the same way, at an
 * address of its own the engine hands here (`batchedTemplate`, `batchedTile`).
 * A billed provider's tile goes through the guard (`quotaGuard.js`), one at a
 * time (`appTile`), and any other at the address it always had.
 */

/** Tiles one request carries (api/tile_batch.py allows 64). */
export const BATCH_MAX = 32;
/** Requests in flight at once, of the browser's six connections to the app. */
export const BATCH_IN_FLIGHT = 4;
/** ms a request waits for the rest of the engine's asks before it goes. */
export const BATCH_WAIT = 4;
/** Bytes of a frame's head: index (u16), status (u16), length (u32). */
const HEAD = 8;

/** Where one tile is asked for on its own: the address it always had. */
function singleUrl({ providerId, z, x, y }) {
  return `/api/tiles/${providerId}/${z}/${x}/${y}`;
}

function abortError() {
  const error = new Error('The tile is no longer wanted');
  error.name = 'AbortError';
  return error;
}

/**
 * Read a batch answer as it arrives, handing over each tile as soon as all of
 * it is in.
 *
 * @param {ReadableStream<Uint8Array>} body
 * @param {(index: number, status: number, data: ArrayBuffer) => void} onTile
 */
export async function readFrames(body, onTile) {
  const reader = body.getReader();
  let held = new Uint8Array(0);
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    const joined = new Uint8Array(held.length + value.length);
    joined.set(held);
    joined.set(value, held.length);
    let at = 0;
    while (joined.length - at >= HEAD) {
      const view = new DataView(joined.buffer, joined.byteOffset + at, HEAD);
      const index = view.getUint16(0, true);
      const status = view.getUint16(2, true);
      const length = view.getUint32(4, true);
      if (joined.length - at - HEAD < length) break;
      const start = at + HEAD;
      onTile(index, status, joined.slice(start, start + length).buffer);
      at = start + length;
    }
    held = joined.slice(at);
  }
}

/**
 * A failed tile as the engine's own loader hands one over, its status and body
 * with it, so the imagery panel reads every failure alike.
 */
function tileError(status, body, statusText = '') {
  const error = new Error(statusText || `tile answered ${status}`);
  error.status = status;
  error.body = body;
  return error;
}

/** The engine's protocol for a flat map's tile asked in batches. */
export const BATCH_PROTOCOL = 'azimut-tiles';

/** Where a flat map asks a provider's tiles to come in batches. */
export function batchedTemplate(providerId) {
  return `${BATCH_PROTOCOL}://${providerId}/{z}/{x}/{y}`;
}

/** The tile a batched address names, or null for an address that is not one. */
export function readBatched(url) {
  const found = new RegExp(`^${BATCH_PROTOCOL}://([^/]+)/(\\d+)/(\\d+)/(\\d+)$`).exec(url);
  return found ? { providerId: found[1], z: Number(found[2]), x: Number(found[3]), y: Number(found[4]) } : null;
}

/**
 * One tile of a flat map through the batches, as the engine wants it back:
 * `{ data }`, or the error its own address would have given (`appTile`).
 *
 * @param {{ providerId: string, z: number, x: number, y: number }} asked
 * @param {object} opts
 * @param {{ get: Function }} opts.batcher `createTileBatcher()`
 * @param {AbortSignal} [opts.signal]
 */
export async function batchedTile({ providerId, z, x, y }, { batcher, signal }) {
  const { status, data } = await batcher.get(`/api/tiles/batch/${providerId}`, z, x, y, signal);
  if (status === 200) return { data };
  throw tileError(status, new Blob([data], { type: 'text/plain; charset=utf-8' }));
}

/**
 * One imagery tile through the app at its own address, as the engine wants it
 * back: `{ data }`, or an error carrying the status and the reason.
 *
 * @param {{ providerId: string, z: number, x: number, y: number }} asked
 * @param {object} [opts]
 * @param {AbortSignal} [opts.signal]
 * @param {typeof fetch} [opts.fetch]
 */
export async function appTile(asked, { signal, fetch: fetcher = (...args) => globalThis.fetch(...args) } = {}) {
  const response = await fetcher(singleUrl(asked), { signal });
  if (!response.ok) throw tileError(response.status, await response.blob(), response.statusText);
  return { data: await response.arrayBuffer() };
}

/**
 * The requests that carry the globe's tiles in batches.
 *
 * `get` answers `{ status, data }` for one tile of one source, or rejects with
 * an AbortError once its signal says the tile is no longer wanted, or with the
 * app's status when it refused the request as a whole. Asks for the
 * same source made within `wait` ms share a request; the same tile asked twice
 * before it goes is asked once.
 *
 * @param {object} [opts]
 * @param {typeof fetch} [opts.fetch]
 * @param {number} [opts.max]
 * @param {number} [opts.inFlight]
 * @param {number} [opts.wait]
 */
export function createTileBatcher({
  fetch: fetcher = (...args) => globalThis.fetch(...args),
  max = BATCH_MAX,
  inFlight = BATCH_IN_FLIGHT,
  wait = BATCH_WAIT,
} = {}) {
  // path → Map(key → [asks]), not sent yet, oldest first
  const pending = new Map();
  let flying = 0;
  let timer = null;

  function settle(ask, fn, value) {
    if (ask.done) return;
    ask.done = true;
    ask.signal?.removeEventListener?.('abort', ask.onAbort);
    fn(value);
  }

  function schedule(now = false) {
    if (now) {
      clearTimeout(timer);
      timer = null;
      flush();
      return;
    }
    if (timer == null) {
      timer = setTimeout(() => {
        timer = null;
        flush();
      }, wait);
    }
  }

  function flush() {
    while (flying < inFlight) {
      const path = [...pending.keys()][0];
      if (path == null) return;
      const asks = pending.get(path);
      const batch = new Map();
      for (const [key, list] of asks) {
        if (batch.size >= max) break;
        batch.set(key, list);
      }
      for (const key of batch.keys()) asks.delete(key);
      if (!asks.size) pending.delete(path);
      // a source with more waiting goes to the back, so one busy source cannot starve another
      else {
        pending.delete(path);
        pending.set(path, asks);
      }
      send(path, batch);
    }
  }

  async function send(path, batch) {
    const keys = [...batch.keys()];
    const lists = [...batch.values()];
    const controller = new AbortController();
    const live = () => lists.some((list) => list.some((ask) => !ask.done));
    for (const list of lists) {
      for (const ask of list) {
        ask.flying = () => {
          if (!live()) controller.abort();
        };
      }
    }
    flying += 1;
    try {
      const response = await fetcher(`${path}?t=${keys.join(',')}`, { signal: controller.signal });
      if (!response.ok || !response.body) {
        const failure = new Error(`The app answered ${response.status}`);
        failure.status = response.status;
        failure.body = await response.blob?.().catch?.(() => null);
        for (const list of lists) for (const ask of list) settle(ask, ask.reject, failure);
        return;
      }
      await readFrames(response.body, (index, status, data) => {
        for (const ask of lists[index] ?? []) settle(ask, ask.resolve, { status, data });
      });
      const missing = new Error('The app left a tile out of its answer');
      for (const list of lists) for (const ask of list) settle(ask, ask.reject, missing);
    } catch (error) {
      for (const list of lists) for (const ask of list) settle(ask, ask.reject, error);
    } finally {
      flying -= 1;
      if (pending.size) schedule(true);
    }
  }

  return {
    /**
     * @param {string} path the source's batch route (`/api/tiles/batch/{provider}`, `/api/terrain/batch`)
     * @param {AbortSignal} [signal]
     */
    get(path, z, x, y, signal) {
      return new Promise((resolve, reject) => {
        if (signal?.aborted) {
          reject(abortError());
          return;
        }
        const key = `${z}/${x}/${y}`;
        const ask = { resolve, reject, signal, done: false, flying: null };
        ask.onAbort = () => {
          settle(ask, ask.reject, abortError());
          const waiting = pending.get(path)?.get(key);
          if (waiting) {
            const left = waiting.filter((other) => other !== ask);
            if (left.length) pending.get(path).set(key, left);
            else pending.get(path).delete(key);
            if (!pending.get(path).size) pending.delete(path);
          }
          ask.flying?.();
        };
        signal?.addEventListener?.('abort', ask.onAbort);
        if (!pending.has(path)) pending.set(path, new Map());
        const asks = pending.get(path);
        asks.set(key, [...(asks.get(key) ?? []), ask]);
        schedule(asks.size >= max);
      });
    },
    /** Requests in flight right now, for a test. */
    get flying() {
      return flying;
    },
  };
}