Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

17.6 KB
Raw Blame
// HTTP wrapper around the core cleaning primitives in server/lib/imageClean.js.
// All sharp/PNG-walker logic lives in the lib so services can call it directly
// (services importing from routes would be a layering violation).
//
// Transport: raw image bytes in the request body (no base64-in-JSON inflation),
// pipeline options in the query string, cleaned bytes back as the response body
// with the small report in the `X-Clean-Report` header. The global
// express.json() parser in server/index.js is a no-op here because the request
// content-type is image/*, so this route's express.raw() owns the body.

import { Router, raw } from 'express';
import { z } from 'zod';
import sharp from 'sharp';
import { asyncHandler, ServerError } from '../lib/errorHandler.js';
import { validateRequest } from '../lib/validation.js';
import { assertHostControl } from '../services/authGate.js';
import { cleanImageBuffer, CLEAN_LEVELS, compositeIgnoreZone, IGNORE_ZONE_FEATHER_DEFAULT } from '../lib/imageClean.js';
import { applyLightRegen, computePixelDelta, REGEN_STRENGTH_MIN, REGEN_STRENGTH_MAX } from '../services/imageGen/regen.js';
import { enqueueGpuClean, readGpuCleanResult, getGpuCleanStatus, saveGpuCleanToGallery } from '../services/imageGen/cleanGpu.js';

const router = Router();

// Re-export so existing `import { CLEAN_LEVELS } from './imageClean.js'`
// consumers in the routes layer keep working.
export { cleanImageBuffer, CLEAN_LEVELS };

// Query-string flag → boolean. Accepts the common truthy/falsey string forms a
// browser fetch sends ("1"/"0"/"true"/"false"); anything else falls back to the
// schema default. Mirrors the history.js success-filter pattern.
const queryFlag = (def) => z.preprocess(
  (v) => {
    if (v === undefined) return def;
    if (v === 'true' || v === '1') return true;
    if (v === 'false' || v === '0') return false;
    return v;
  },
  z.boolean(),
);

// Diffusion sub-mode. `off` (default) skips it entirely; `light` runs the
// CPU-only spatial pass (`applyLightRegen`) that actually perturbs SynthID's
// resolution-dependent carriers — the no-GPU fallback, always available since
// sharp is a hard dependency. `gpu` (the FLUX img2img round-trip) is
// GPU-serialized via mediaJobQueue: the route stages the sync-cleaned bytes,
// enqueues a job through the shared regen seam, and returns a `jobId` the
// client tracks via the media-job progress channel (issue #2264). Anything
// else → the `off` default.
const diffusionMode = z.preprocess(
  (v) => {
    if (v === undefined || v === '' || v === '0' || v === 'false' || v === 'off') return 'off';
    if (v === 'light' || v === 'cpu' || v === '1' || v === 'true') return 'light';
    if (v === 'gpu' || v === 'flux') return 'gpu';
    return 'off';
  },
  z.enum(['off', 'light', 'gpu']),
);

// Pipeline steps ride in the query string, not the body (the body is the image).
// `mask=1` signals an ignore-zone preserve-region envelope in the body (see the
// length-prefixed decode below); `feather` is the soft-boundary blur sigma (px)
// clamped in the lib. The mask only has any effect when a diffusion step runs —
// it composites the ORIGINAL pixels back over the diffused result.
const cleanQuerySchema = z.object({
  metadata: queryFlag(true),
  denoise: queryFlag(false),
  diffusion: diffusionMode,
  mask: queryFlag(false),
  feather: z.preprocess(
    (v) => (v === undefined || v === '' ? IGNORE_ZONE_FEATHER_DEFAULT : Number(v)),
    z.number().min(0).max(50),
  ),
  // GPU-diffusion-only knobs (issue #2264), ignored by the sync passes.
  // `strength` is the img2img denoise, clamped to the same [MIN, MAX] the
  // lightbox slider is bound to; absent → the conservative arbitrary-upload
  // default. `maxMp` is the optional per-run render-megapixel budget override.
  strength: z.preprocess(
    (v) => (v === undefined || v === '' ? undefined : Number(v)),
    z.number().min(REGEN_STRENGTH_MIN).max(REGEN_STRENGTH_MAX).optional(),
  ),
  maxMp: z.preprocess(
    (v) => (v === undefined || v === '' ? undefined : Number(v)),
    z.number().min(0.25).max(16).optional(),
  ),
});

// Decode the optional ignore-zone envelope. When `?mask=1`, the body is
// `<uint32 BE maskLen><mask PNG bytes><image bytes>` — a no-dependency framing
// that keeps the whole payload inside express.raw() without pulling in
// multer/busboy for a second multipart field. Returns `{ image, mask }` (mask
// null when absent/malformed — the pipeline just skips the composite, never
// 500s). The mask is small (a 1-channel PNG), so the 4-byte length prefix is
// plenty. Guards against a length that overruns the buffer.
function splitMaskEnvelope(body, hasMask) {
  if (!Buffer.isBuffer(body) || body.length === 0) return { image: null, mask: null };
  if (!hasMask) return { image: body, mask: null };
  if (body.length < 4) return { image: body, mask: null };
  const maskLen = body.readUInt32BE(0);
  const maskEnd = 4 + maskLen;
  // A length that doesn't leave any image bytes (or overruns) is malformed —
  // fall back to treating the whole body as the image so a bad frame degrades to
  // "no mask" rather than a hard failure.
  if (maskLen <= 0 || maskEnd >= body.length) return { image: body, mask: null };
  return { image: body.subarray(maskEnd), mask: body.subarray(4, maskEnd) };
}

// 256 MiB raw-byte ceiling — the transport is the only (generous) size bound now
// that base64 inflation is gone. The decompression-bomb guard (MAX_PIXELS) in
// the lib still protects the process regardless of byte size.
const RAW_LIMIT = '256mb';
// Include the common JPEG MIME aliases (image/jpg, image/pjpeg) so direct API
// callers that send them get parsed; the bundled frontend forces
// application/octet-stream, but other clients shouldn't have to. Format is
// authoritatively decided by the magic-byte sniff, not the content-type.
const RAW_TYPES = ['image/png', 'image/jpeg', 'image/jpg', 'image/pjpeg', 'image/webp', 'application/octet-stream'];

router.post(
  '/',
  raw({ type: RAW_TYPES, limit: RAW_LIMIT }),
  asyncHandler(async (req, res) => {
    const { metadata, denoise, diffusion, mask: hasMask, feather, strength, maxMp } = validateRequest(cleanQuerySchema, req.query);
    // The GPU pass queues a local Python FLUX job (#13055); the sync passes are
    // in-process sharp and stay open. The mode rides in the query, which the
    // body gate cannot see, so the handler checks it.
    if (diffusion === 'gpu') assertHostControl(req);

    const rawBody = Buffer.isBuffer(req.body) ? req.body : null;
    const { image: buffer, mask: maskBuffer } = splitMaskEnvelope(rawBody, hasMask);
    if (!buffer || buffer.length === 0) {
      throw new ServerError('Request body must be raw image bytes', {
        status: 400,
        code: 'VALIDATION_ERROR',
      });
    }

    // The original bytes are captured BEFORE any cleaning so the ignore-zone
    // composite can restore true source pixels into the preserved regions (the
    // whole point is to undo the diffusion inside the mask, not re-diffuse it).
    let originalBuffer = buffer;

    const result = await cleanImageBuffer(buffer, { metadata, denoise });

    // GPU FLUX round-trip (issue #2264). Unlike the sync passes, the GPU render
    // is GPU-serialized through mediaJobQueue and can't run inline — so hand the
    // already-sync-cleaned bytes to the shared regen seam, enqueue a job, and
    // return a `jobId` (HTTP 202). The client tracks progress via the media-job
    // channel, then GETs the finished bytes from the result endpoint. The mask,
    // when present, is staged with the (oriented) original so the result-fetch
    // composites the preserved region back over the diffused frame.
    if (diffusion === 'gpu') {
      // Orientation parity (same rationale as the light-pass block below): bake
      // EXIF orientation into the init + preserved-original so the mask (painted
      // in visual space) aligns with the render.
      let gpuInit = result.data;
      let gpuOriginal = maskBuffer ? originalBuffer : null;
      // Dimensions the render + upscale-back target. `result.width/height` are
      // the (metadata-step) cleaned dims; but when a mask forces the oriented
      // re-encode below, the oriented buffer's dims can differ (EXIF 6/8 swaps
      // W/H) — so read them off the ORIENTED buffer, or FLUX renders at the
      // wrong aspect and the composite misaligns.
      let sourceDims = result.width && result.height ? { width: result.width, height: result.height } : null;
      if (maskBuffer) {
        const oriented = await sharp(result.data).rotate().png().toBuffer({ resolveWithObject: true }).catch(() => null);
        if (oriented) {
          gpuInit = oriented.data;
          gpuOriginal = oriented.data;
          sourceDims = { width: oriented.info.width, height: oriented.info.height };
        }
      }
      const queued = await enqueueGpuClean({
        initBuffer: gpuInit,
        sourceDims,
        strength,
        maxMegapixels: maxMp,
        originalBuffer: maskBuffer ? gpuOriginal : null,
        maskBuffer: maskBuffer || null,
        feather,
      });
      console.log(`🧼 Image clean GPU job queued: ${queued.modelId} (strength=${queued.strength}) → job ${queued.jobId.slice(0, 8)}`);
      return res.status(202).json({
        mode: 'gpu',
        ...queued,
        c2paStripped: result.c2paStripped,
        c2paPresent: result.c2paPresent,
        steps: result.steps,
      });
    }

    // Diffusion (CPU light pass) runs LAST in the pipeline order (metadata →
    // denoise → diffusion), on the already-cleaned bytes. `applyLightRegen` is a
    // pure sharp round-trip (resize-squeeze + micro color nudge + high-freq
    // perturbation) that perturbs SynthID's resolution-dependent carriers — the
    // only step here that touches SynthID at all, and honestly best-effort. It
    // always re-encodes to PNG, so the output format/mime flip to PNG when it
    // runs (regardless of the source format). No gallery/GPU dependency.
    let outData = result.data;
    let outFormat = result.format;
    let outMime = result.mimeType;
    let outWidth = result.width;
    let outHeight = result.height;
    const steps = [...result.steps];

    if (diffusion === 'light') {
      // Orientation parity for the ignore-zone composite: the user paints the
      // mask on the browser preview (EXIF-oriented visual space), but when the
      // metadata step is OFF `cleanImageBuffer` returns the source bytes with
      // their orientation tag intact, and `applyLightRegen` reads UN-oriented
      // dims. Left as-is, the mask (oriented) and the diffused base (stored
      // space) would misalign under the composite's fit:fill resize on an
      // Orientation 6/8 source. Bake orientation into BOTH the diffusion input
      // and the preserved-original buffer so base + original + mask all share
      // one visual space. No-op for the common already-oriented/no-orientation
      // cases (a `.rotate()` re-encode with no EXIF tag just re-emits pixels).
      let diffusionInput = result.data;
      if (maskBuffer) {
        const orientedPng = await sharp(result.data).rotate().png().toBuffer().catch(() => null);
        if (orientedPng) {
          diffusionInput = orientedPng;
          originalBuffer = orientedPng;
        }
      }
      const light = await applyLightRegen(diffusionInput);
      if (!light) {
        throw new ServerError('Invalid or corrupt image', {
          status: 400,
          code: 'INVALID_IMAGE',
        });
      }
      // Fidelity metric so the report can show how much the pass actually moved
      // the pixels (the pixel-delta/PSNR the diffusion research gates on) —
      // "disrupt", never "remove". Skipped silently on a decode failure.
      const delta = await computePixelDelta(result.data, light.data).catch(() => null);
      outData = light.data;
      outFormat = 'png';
      outMime = 'image/png';
      outWidth = light.width;
      outHeight = light.height;
      steps.push({
        step: 'diffusion',
        status: 'applied',
        lossless: false,
        mode: 'light',
        // ASCII-only: this rides in the X-Clean-Report HTTP header, which is
        // latin1 — a stray em-dash or delta glyph makes Node reject the send.
        detail: delta
          ? `CPU light pass - best-effort SynthID disruption (pixel delta ${delta.pixelDeltaPct}%, PSNR ${delta.psnr}dB); does not guarantee removal`
          : 'CPU light pass - best-effort SynthID disruption; does not guarantee removal',
        ...(delta ? { pixelDeltaPct: delta.pixelDeltaPct, psnr: delta.psnr } : {}),
      });

      // Ignore-zone (preserve-region) compositing runs LAST, only after a
      // diffusion pass has actually redrawn the frame. It restores the ORIGINAL
      // pixels into the user-painted mask (feathered edge) so comic dialog /
      // faces / fine text the diffusion garbled are preserved — a deliberate
      // per-region quality-vs-disruption choice (those regions keep their local
      // SynthID). A missing/malformed mask or a decode failure degrades to the
      // un-composited diffused bytes rather than failing the request.
      if (maskBuffer) {
        const composited = await compositeIgnoreZone(outData, originalBuffer, maskBuffer, { feather });
        if (composited) {
          outData = composited.data;
          outFormat = 'png';
          outMime = 'image/png';
          outWidth = composited.width;
          outHeight = composited.height;
          steps.push({
            step: 'ignore-zone',
            status: 'applied',
            lossless: false,
            detail: `preserve-region composite (feather ${feather}px) - masked pixels keep their original SynthID`,
          });
        } else {
          steps.push({
            step: 'ignore-zone',
            status: 'noop',
            lossless: false,
            detail: 'mask could not be applied (decode failed) - returned the un-preserved diffused result',
          });
        }
      }
    }

    const report = {
      format: outFormat,
      sizeBefore: result.sizeBefore,
      sizeAfter: outData.length,
      width: outWidth,
      height: outHeight,
      c2paStripped: result.c2paStripped,
      c2paPresent: result.c2paPresent,
      steps,
    };

    console.log(`🧼 Image cleaned: ${outFormat} ${result.sizeBefore}B → ${outData.length}B (c2pa=${result.c2paStripped}, steps=${steps.map((s) => s.step).join('+') || 'none'})`);

    // Expose the report header so a cross-origin / dev-proxied fetch can read it.
    res.setHeader('Access-Control-Expose-Headers', 'X-Clean-Report');
    res.setHeader('X-Clean-Report', JSON.stringify(report));
    res.setHeader('Content-Type', outMime);
    res.send(outData);
  }),
);

// Validate a clean job id — the mediaJobQueue mints v4 UUIDs, so anything that
// isn't UUID-shaped can't be a real job (and keeps the temp-path resolution
// safe). Reused by the result-fetch + save-to-gallery routes below.
const jobIdSchema = z.object({
  jobId: z.string().uuid('Invalid job id'),
});

// Result-fetch for a GPU FLUX clean job (issue #2264). Returns the finished
// temp render bytes (with any pending ignore-zone composite applied) as the
// response body — the default is NOT to keep them in the gallery. 409 while the
// job is still queued/running (the client keeps polling the media-job channel);
// 404 once past the temp GC / never produced. The result stays ephemeral: the
// caller saves it explicitly via POST below if they want to keep it.
router.get(
  '/result/:jobId',
  asyncHandler(async (req, res) => {
    const { jobId } = validateRequest(jobIdSchema, req.params);
    const status = getGpuCleanStatus(jobId);
    if (status.status === 'failed' || status.status === 'canceled') {
      throw new ServerError(status.error || `Clean job ${status.status}`, { status: 409, code: 'JOB_FAILED' });
    }
    // Never read bytes off disk while the job is still queued/running: the
    // runner writes the render mid-flight (and the completion path upscales it
    // back to source dims AFTER the initial write), so a read before the
    // terminal `completed` state could return a partial or pre-upscale frame.
    // 409 keeps the client polling. (A job that vanished from the queue's 24h
    // archive reads as `unknown` — fall through and let the disk decide 200 vs
    // 404, so a completed-then-forgotten render is still fetchable.)
    if (status.status === 'queued' || status.status === 'running') {
      throw new ServerError('Clean result not ready yet', { status: 409, code: 'RESULT_NOT_READY', severity: 'warning' });
    }
    const result = await readGpuCleanResult(jobId);
    if (!result) {
      // Completed/unknown but no bytes on disk → gone (past the temp GC or an
      // unknown id). A true miss so the client stops polling.
      throw new ServerError('Clean result not found', { status: 404, code: 'NOT_FOUND', severity: 'warning' });
    }
    res.setHeader('Access-Control-Expose-Headers', 'X-Clean-Report');
    res.setHeader('X-Clean-Report', JSON.stringify({
      format: 'png',
      width: result.width,
      height: result.height,
      composited: result.composited,
      mode: 'gpu',
    }));
    res.setHeader('Content-Type', 'image/png');
    res.send(result.data);
  }),
);

// Explicit save-to-gallery for a GPU FLUX clean result (issue #2264). The
// default clean flow never touches the gallery; this promotes the finished
// temp render to a first-class gallery citizen (reusing saveUploadedGalleryImage
// so it lists/deletes like any other). Returns the new gallery `{ filename, path }`.
router.post(
  '/result/:jobId/save',
  asyncHandler(async (req, res) => {
    const { jobId } = validateRequest(jobIdSchema, req.params);
    const saved = await saveGpuCleanToGallery(jobId);
    res.status(201).json(saved);
  }),
);

export default router;