Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
2.8 KB
Raw Blame
/**
 * spool.ts — transient spool of fetched conversations in native-export shape.
 *
 * The connector writes fetched conversations to a file shaped exactly like the
 * provider's native export (a top-level JSON array), then hands the path to
 * `runTranscriptsIngest`. The spool holds UNREDACTED text, so it is written
 * 0600 under the 0700 connectors dir and pruned immediately after ingest
 * (`finally` in the orchestrator), not left lying around. Batched writes keep a
 * multi-thousand-thread `--full` backfill memory-bounded and bank progress.
 */

import { chmodSync, mkdirSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { connectorsDir } from './credentials.ts';
import type { ConnectorProviderName } from './types.ts';

/** Batch size for spool-then-ingest chunking (memory bound on `--full`). */
export const CONNECTOR_SPOOL_BATCH = 200;

export function spoolDir(provider: ConnectorProviderName): string {
  return join(connectorsDir(), 'spool', provider);
}

function ensureSpoolDir(provider: ConnectorProviderName): string {
  const dir = spoolDir(provider);
  mkdirSync(dir, { recursive: true });
  try {
    chmodSync(join(connectorsDir(), 'spool'), 0o700);
    chmodSync(dir, 0o700);
  } catch {
    // best-effort
  }
  return dir;
}

/**
 * Write a batch of native-export-shaped conversation objects to a spool file
 * (0600) and return its path. `stamp` must be caller-provided (a timestamp or
 * batch ordinal) — this module never reads the clock.
 */
export function writeSpool(
  provider: ConnectorProviderName,
  conversations: Array<Record<string, unknown>>,
  stamp: string,
): string {
  const dir = ensureSpoolDir(provider);
  const target = join(dir, `${stamp}.json`);
  const tmp = `${target}.tmp-${process.pid}`;
  writeFileSync(tmp, JSON.stringify(conversations), { mode: 0o600 });
  renameSync(tmp, target);
  try {
    chmodSync(target, 0o600);
  } catch {
    // best-effort
  }
  return target;
}

/** Delete a single spool file (best-effort; called in the orchestrator finally). */
export function removeSpool(path: string): void {
  try {
    rmSync(path);
  } catch {
    // best-effort
  }
}

/**
 * Prune the provider's spool dir to the newest `keep` files (default 0 = clear
 * all). Filenames are the caller's monotonic stamps, so lexical sort == age.
 */
export function pruneSpool(provider: ConnectorProviderName, keep = 0): number {
  let removed = 0;
  try {
    const dir = spoolDir(provider);
    const files = readdirSync(dir)
      .filter((f) => f.endsWith('.json'))
      .sort();
    const doomed = keep > 0 ? files.slice(0, Math.max(0, files.length - keep)) : files;
    for (const f of doomed) {
      try {
        rmSync(join(dir, f));
        removed++;
      } catch {
        // best-effort
      }
    }
  } catch {
    // dir missing → nothing to prune
  }
  return removed;
}