Skip to content

LIBERATED AI CHAT

TypeScript
10.0 KB
Raw Blame
/**
 * Research API Routes
 *
 * Read-access endpoints for the pliny-the-prompter/g0dm0d3 HuggingFace
 * dataset. Lets researchers query, filter, and download the full
 * published corpus — not just the current in-memory buffer.
 *
 * GET  /v1/research/info       — Dataset repo info + schema
 * GET  /v1/research/stats      — Aggregate stats across all published batches
 * GET  /v1/research/batches    — List all published batch files
 * GET  /v1/research/batch/:path — Read a single batch file
 * GET  /v1/research/query      — Query the full corpus with filters
 * POST /v1/research/flush      — Force-flush in-memory buffers to HF
 * GET  /v1/research/download   — Download the full corpus as JSONL
 */

import { Router } from 'express'
import {
  isReaderEnabled,
  listBatchFiles,
  readBatch,
  queryCorpus,
  getCorpusStats,
  flushMetadata,
  flushDataset,
} from '../lib/hf-reader'
import { getDataset, getDatasetStats } from '../lib/dataset'
import { getStats as getMetadataStats } from '../lib/metadata'
import { getPublisherStatus } from '../lib/hf-publisher'
import { tierGate } from '../middleware/tierGate'

export const researchRoutes = Router()

let lastFlushTime = 0

// ── Info: dataset schema, repo, capabilities ─────────────────────────

researchRoutes.get('/info', (_req, res) => {
  res.json({
    name: 'G0DM0D3 Research Dataset',
    repo: 'pliny-the-prompter/g0dm0d3',
    repo_url: 'https://huggingface.co/datasets/pliny-the-prompter/g0dm0d3',
    license: 'AGPL-3.0',
    description: 'Open research dataset for LLM robustness and safety evaluation. Contains opt-in interaction data with full pipeline metadata from ULTRAPLINIAN multi-model racing, AutoTune context-adaptive parameter tuning, Parseltongue obfuscation, and STM output normalization.',
    hf_reader_enabled: isReaderEnabled(),
    publisher: getPublisherStatus(),
    schema: {
      dataset_entry: {
        id: 'string (UUID)',
        timestamp: 'number (unix ms)',
        endpoint: 'string',
        model: 'string (OpenRouter model ID)',
        mode: '"standard" | "ultraplinian"',
        messages: 'Array<{ role: string, content: string }>',
        response: 'string',
        autotune: '{ strategy, detected_context, confidence, params, reasoning } | null',
        parseltongue: '{ triggers_found, technique_used, transformations_count } | null',
        stm: '{ modules_applied: string[] } | null',
        ultraplinian: '{ tier, models_queried, winner_model, all_scores, total_duration_ms } | null',
        feedback: '{ rating: 1|-1, heuristics } | null',
      },
      metadata_event: {
        id: 'string (UUID)',
        timestamp: 'number (unix ms)',
        endpoint: 'string',
        mode: '"standard" | "ultraplinian"',
        tier: 'string | null',
        stream: 'boolean',
        pipeline: '{ godmode, autotune, parseltongue, stm_modules, strategy }',
        autotune: '{ detected_context, confidence } | null',
        model_results: 'Array<{ model, score, duration_ms, success, content_length, error_type }> | null',
        winner: '{ model, score, duration_ms, content_length } | null',
        total_duration_ms: 'number',
        response_length: 'number',
        liquid: '{ upgrades, first_response_ms } | null',
      },
    },
    endpoints: {
      'GET /v1/research/info': 'This endpoint — dataset schema and repo info',
      'GET /v1/research/stats': 'Aggregate stats across all published HF batches',
      'GET /v1/research/batches': 'List all published batch files in the HF repo',
      'GET /v1/research/batch/*': 'Read a specific batch file (JSONL parsed)',
      'GET /v1/research/query': 'Query the full corpus with filters (category, model, mode, time range)',
      'POST /v1/research/flush': 'Force-flush in-memory buffers to HuggingFace',
      'GET /v1/research/download': 'Download full corpus as streaming JSONL',
      'GET /v1/research/combined-stats': 'Combined view: in-memory + published stats',
    },
  })
})

// ── Stats: aggregate across all published batches ────────────────────

researchRoutes.get('/stats', async (_req, res) => {
  try {
    const stats = await getCorpusStats()
    res.json(stats)
  } catch (err) {
    console.error('[research]', err)
    res.status(500).json({ error: 'Failed to fetch corpus stats' })
  }
})

// ── Combined Stats: in-memory + published ────────────────────────────

researchRoutes.get('/combined-stats', async (_req, res) => {
  try {
    const [corpusStats, datasetStats, metadataStats] = await Promise.all([
      getCorpusStats(),
      Promise.resolve(getDatasetStats()),
      Promise.resolve(getMetadataStats()),
    ])

    res.json({
      published: corpusStats,
      in_memory: {
        dataset: datasetStats,
        metadata: {
          total_requests: metadataStats.total_requests,
          by_mode: metadataStats.by_mode,
          models: metadataStats.models,
          latency: metadataStats.latency,
          errors: metadataStats.errors,
        },
      },
    })
  } catch (err) {
    console.error('[research]', err)
    res.status(500).json({ error: 'Failed to fetch combined stats' })
  }
})

// ── Batches: list all published files ────────────────────────────────

researchRoutes.get('/batches', async (req, res) => {
  try {
    const refresh = req.query.refresh === 'true'
    const category = req.query.category as 'metadata' | 'dataset' | undefined

    let batches = await listBatchFiles(refresh)

    if (category) {
      batches = batches.filter(b => b.category === category)
    }

    res.json({
      batches,
      total: batches.length,
      hf_reader_enabled: isReaderEnabled(),
    })
  } catch (err) {
    console.error('[research]', err)
    res.status(500).json({ error: 'Failed to list batches' })
  }
})

// ── Batch: read a single file ────────────────────────────────────────
// Path comes as wildcard: /v1/research/batch/metadata/batch_2024-01-01_0001.jsonl

researchRoutes.get('/batch/*', async (req, res) => {
  try {
    const filePath = req.params[0]
    if (!filePath || !filePath.endsWith('.jsonl')) {
      res.status(400).json({ error: 'Invalid batch path. Expected: metadata/batch_*.jsonl or dataset/batch_*.jsonl' })
      return
    }

    if (!/^(metadata|dataset)\/batch_[\w.-]+\.jsonl$/.test(filePath)) {
      res.status(400).json({ error: 'Invalid batch path format' })
      return
    }

    const records = await readBatch(filePath)
    res.json({
      path: filePath,
      records,
      count: records.length,
    })
  } catch (err) {
    console.error('[research]', err)
    res.status(500).json({ error: 'Failed to read batch' })
  }
})

// ── Query: search the full corpus ────────────────────────────────────

researchRoutes.get('/query', tierGate('research:full'), async (req, res) => {
  try {
    const query = {
      category: req.query.category as 'metadata' | 'dataset' | undefined,
      since: req.query.since ? parseInt(String(req.query.since)) : undefined,
      until: req.query.until ? parseInt(String(req.query.until)) : undefined,
      model: req.query.model as string | undefined,
      mode: req.query.mode as 'standard' | 'ultraplinian' | undefined,
      limit: Math.min(parseInt(String(req.query.limit)) || 100, 1000),
      offset: parseInt(String(req.query.offset)) || 0,
    }

    const result = await queryCorpus(query)
    res.json({
      ...result,
      query,
      has_more: result.truncated,
    })
  } catch (err) {
    console.error('[research]', err)
    res.status(500).json({ error: 'Failed to query corpus' })
  }
})

// ── Flush: force-push in-memory buffers to HF ───────────────────────

researchRoutes.post('/flush', tierGate('flush'), async (_req, res) => {
  if (Date.now() - lastFlushTime < 300_000) {
    res.status(429).json({ error: 'Flush cooldown: please wait 5 minutes between flushes' })
    return
  }

  try {
    lastFlushTime = Date.now()
    const [meta, data] = await Promise.all([
      flushMetadata(),
      flushDataset(),
    ])

    res.json({
      metadata: meta,
      dataset: data,
      message: meta.success || data.success
        ? `Flushed ${meta.flushed} metadata events + ${data.flushed} dataset entries to HuggingFace`
        : 'Nothing to flush (buffers empty or HF publishing not configured)',
    })
  } catch (err) {
    console.error('[research]', err)
    res.status(500).json({ error: 'Flush failed' })
  }
})

// ── Download: stream the full corpus as JSONL ────────────────────────

researchRoutes.get('/download', tierGate('corpus:download'), async (req, res) => {
  try {
    const category = req.query.category as 'metadata' | 'dataset' | undefined
    const includeMemory = req.query.include_memory !== 'false' // default: true

    res.setHeader('Content-Type', 'application/x-ndjson')
    res.setHeader('Content-Disposition', `attachment; filename="g0dm0d3-corpus-${Date.now()}.jsonl"`)

    // First: stream published batches from HF
    const batches = await listBatchFiles()
    const targetBatches = category ? batches.filter(b => b.category === category) : batches

    for (const batch of targetBatches) {
      const records = await readBatch(batch.path)
      for (const record of records) {
        res.write(JSON.stringify(record) + '\n')
      }
    }

    // Then: append current in-memory data (not yet published)
    if (includeMemory) {
      const memoryDataset = getDataset()
      for (const entry of memoryDataset) {
        res.write(JSON.stringify({ ...entry, _source: 'memory' }) + '\n')
      }
    }

    res.end()
  } catch (err) {
    // If headers already sent, can't send JSON error
    console.error('[research]', err)
    if (!res.headersSent) {
      res.status(500).json({ error: 'Download failed' })
    } else {
      res.end()
    }
  }
})