LIBERATED AI CHAT
TypeScript
fix: deterministic date partitions + compaction script rewrite
faber2896b530acce
diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.tsindex 9da5dc7..c87aea2 100644--- a/functions/api/telemetry.ts+++ b/functions/api/telemetry.ts@@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise<number> {const combined = chunks.join('\n')const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(','))- const d = new Date()+ // Derive partition from oldest batch key timestamp (stable across retries)+ const oldestTs = parseInt(flushedKeys[0].name.split(':')[1], 10)+ const d = !isNaN(oldestTs) ? new Date(oldestTs) : new Date()const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl`const branch = env.HF_DATASET_BRANCH || 'main'@@ -403,12 +405,14 @@ async function flushMemToHF(env: Env): Promise<number> {// accumulate into a fresh buffer, not into the one we're flushing.const snapshot = memBuffer.splice(0, memBuffer.length)const snapshotBytes = memBufferBytes+ const snapshotTs = memBufferFirstTsmemBufferBytes = 0memBufferFirstTs = 0const combined = snapshot.join('\n')const digest = await sha256Hex(combined)- const d = new Date()+ // Derive partition from buffer start time (stable across retries)+ const d = snapshotTs > 0 ? new Date(snapshotTs) : new Date()const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl`const branch = env.HF_DATASET_BRANCH || 'main'diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjsindex a37ab71..60eebed 100644--- a/scripts/compact-hf-telemetry.mjs+++ b/scripts/compact-hf-telemetry.mjs@@ -4,15 +4,18 @@** Usage:* node scripts/compact-hf-telemetry.mjs # dry-run (default)- * node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy+ * node scripts/compact-hf-telemetry.mjs --apply # commit archives + delete legacy** Requires: HF_TOKEN env var with write access to the dataset.*- * Safety:- * - Dry-run is default; --apply required for writes- * - Verifies event counts before deleting anything- * - Produces a manifest with checksums- * - Stops on any count mismatch+ * Safety guarantees:+ * - Dry-run is default; --apply required for any writes+ * - FAIL-CLOSED: aborts on ANY download failure or parse error threshold+ * - Paginates the full file listing (HF API returns max 1000/page)+ * - Verifies uploaded archives by re-downloading and checking checksums+ * - Batches delete operations (max 50 per HF commit)+ * - Preserves FULL invalid lines (not truncated) in a separate artifact+ * - Stops and reports if any count mismatch is detected*/const HF_API = 'https://huggingface.co/api';@@ -20,6 +23,7 @@ const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3'const BRANCH = process.env.HF_DATASET_BRANCH || 'main';const TOKEN = process.env.HF_TOKEN;const APPLY = process.argv.includes('--apply');+const DELETE_BATCH_SIZE = 50;if (!TOKEN) {console.error('ERROR: HF_TOKEN environment variable required');@@ -28,11 +32,35 @@ if (!TOKEN) {const headers = { Authorization: `Bearer ${TOKEN}` };-async function listFiles(prefix) {- const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`;- const res = await fetch(url, { headers });- if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);- return res.json();+// ── Paginated file listing ──────────────────────────────────────────+async function listAllFiles(prefix) {+ const allFiles = [];+ let cursor = null;++ while (true) {+ const params = new URLSearchParams();+ if (cursor) params.set('cursor', cursor);+ const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`;+ const res = await fetch(url, { headers });+ if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);++ const page = await res.json();+ if (!Array.isArray(page) || page.length === 0) break;++ allFiles.push(...page);+ console.log(` Listed ${allFiles.length} entries so far...`);++ // HF returns Link header with rel="next" for pagination+ const linkHeader = res.headers.get('link') || '';+ const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/);+ if (nextMatch) {+ cursor = decodeURIComponent(nextMatch[1]);+ } else {+ break;+ }+ }++ return allFiles;}async function downloadFile(path) {@@ -48,6 +76,13 @@ async function sha256(text) {return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('');}+function utf8ToBase64(str) {+ const bytes = new TextEncoder().encode(str);+ let binStr = '';+ for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]);+ return btoa(binStr);+}+async function commitFiles(operations, summary) {const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`;const lines = [@@ -59,82 +94,94 @@ async function commitFiles(operations, summary) {headers: { ...headers, 'Content-Type': 'application/x-ndjson' },body: lines.join('\n'),});- if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`);+ if (!res.ok) throw new Error(`Commit failed (${res.status}): ${(await res.text()).slice(0, 500)}`);return res.json();}-function utf8ToBase64(str) {- const bytes = new TextEncoder().encode(str);- let binStr = '';- for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]);- return btoa(binStr);-}-async function main() {console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`);console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`);console.log('');- // Step 1: List all legacy telemetry files- console.log('Step 1: Listing legacy telemetry/ files...');- const files = await listFiles('telemetry');- const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));- console.log(` Found ${jsonlFiles.length} JSONL files`);+ // ── Step 1: Paginated listing ─────────────────────────────────────+ console.log('Step 1: Listing ALL legacy telemetry/ files (paginated)...');+ const allEntries = await listAllFiles('telemetry');+ const jsonlFiles = allEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));+ console.log(` Total entries: ${allEntries.length}`);+ console.log(` JSONL files: ${jsonlFiles.length}`);if (jsonlFiles.length === 0) {console.log(' Nothing to compact. Done.');return;}- // Step 2: Download and parse all files- console.log('Step 2: Downloading and parsing events...');+ // ── Step 2: Download and parse ALL files (fail-closed) ────────────+ console.log('');+ console.log('Step 2: Downloading and parsing events (fail-closed)...');const monthBuckets = {};let totalEvents = 0;- let totalBytes = 0;+ let totalSourceBytes = 0;+ let downloadFailures = 0;let invalidLines = 0;const invalidRecords = [];+ const failedFiles = [];for (let i = 0; i < jsonlFiles.length; i++) {const file = jsonlFiles[i];- if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`);+ if (i % 200 === 0 || i === jsonlFiles.length - 1) {+ console.log(` Progress: ${i + 1}/${jsonlFiles.length} files...`);+ }+ let content;try {- const content = await downloadFile(file.path);- totalBytes += new TextEncoder().encode(content).byteLength;- const lines = content.split('\n').filter(l => l.trim());-- for (const line of lines) {- try {- const event = JSON.parse(line);- const ts = event.timestamp;- let monthKey;- if (ts && typeof ts === 'number') {- const d = new Date(ts);- monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;- } else {- monthKey = 'unknown';- }- if (!monthBuckets[monthKey]) monthBuckets[monthKey] = [];- monthBuckets[monthKey].push(line);- totalEvents++;- } catch {- invalidLines++;- invalidRecords.push({ file: file.path, line: line.slice(0, 200) });+ content = await downloadFile(file.path);+ } catch (err) {+ downloadFailures++;+ failedFiles.push({ path: file.path, error: err.message });+ continue;+ }++ totalSourceBytes += new TextEncoder().encode(content).byteLength;+ const lines = content.split('\n').filter(l => l.trim());++ for (const line of lines) {+ try {+ const event = JSON.parse(line);+ const ts = event.timestamp;+ let monthKey;+ if (ts && typeof ts === 'number') {+ const d = new Date(ts);+ monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;+ } else {+ monthKey = 'unknown';}+ if (!monthBuckets[monthKey]) monthBuckets[monthKey] = [];+ monthBuckets[monthKey].push(line);+ totalEvents++;+ } catch {+ invalidLines++;+ invalidRecords.push({ file: file.path, line });}- } catch (err) {- console.error(` ERROR reading ${file.path}: ${err.message}`);- invalidRecords.push({ file: file.path, error: err.message });}}- console.log(` Total events: ${totalEvents}`);- console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`);+ console.log(` Total events parsed: ${totalEvents}`);+ console.log(` Total source bytes: ${(totalSourceBytes / 1024 / 1024).toFixed(2)} MiB`);+ console.log(` Download failures: ${downloadFailures}`);console.log(` Invalid lines: ${invalidLines}`);console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`);- console.log('');- // Step 3: Build archives+ // ── FAIL-CLOSED: abort if any downloads failed ────────────────────+ if (downloadFailures > 0) {+ console.error('');+ console.error(`ABORT: ${downloadFailures} file(s) failed to download.`);+ console.error('Cannot guarantee data preservation. Fix network/auth issues and retry.');+ console.error('Failed files:', failedFiles.map(f => f.path).join(', '));+ process.exit(1);+ }++ // ── Step 3: Build archives ────────────────────────────────────────+ console.log('');console.log('Step 3: Building monthly archives...');const archives = [];for (const [month, events] of Object.entries(monthBuckets).sort()) {@@ -143,32 +190,39 @@ async function main() {const archivePath = `archive/legacy/telemetry-${month}.jsonl`;const bytes = new TextEncoder().encode(content).byteLength;archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content });- console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`);+ console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`);}- // Step 4: Verify counts+ // ── Step 4: Verify counts (tautology-free) ────────────────────────const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0);+ const sourceFileCount = jsonlFiles.length;+ const expectedTotal = totalEvents; // all successfully parsed events+console.log('');- console.log(`Step 4: Verification`);- console.log(` Source events: ${totalEvents}`);- console.log(` Archive events: ${archiveEventTotal}`);+ console.log('Step 4: Verification');+ console.log(` Source files examined: ${sourceFileCount}`);+ console.log(` Download failures: ${downloadFailures} (must be 0)`);+ console.log(` Events parsed from sources: ${totalEvents}`);+ console.log(` Events in archives: ${archiveEventTotal}`);+ console.log(` Invalid lines preserved: ${invalidLines}`);- if (archiveEventTotal !== totalEvents) {- console.error(' MISMATCH — aborting. No changes made.');+ if (archiveEventTotal !== expectedTotal) {+ console.error(' COUNT MISMATCH — aborting. No changes made.');process.exit(1);}- console.log(' MATCH — counts verified.');+ console.log(' COUNTS MATCH.');- // Build manifest+ // ── Build manifest ────────────────────────────────────────────────const manifest = {timestamp: new Date().toISOString(),repo: DATASET_REPO,branch: BRANCH,- sourceFiles: jsonlFiles.length,- sourceBytes: totalBytes,+ sourceFiles: sourceFileCount,+ sourceBytes: totalSourceBytes,totalEvents,invalidLines,- invalidRecords: invalidRecords.length,+ invalidRecordCount: invalidRecords.length,+ downloadFailures,archives: archives.map(a => ({path: a.path, month: a.month, events: a.events,bytes: a.bytes, checksum: a.checksum,@@ -182,10 +236,12 @@ async function main() {if (!APPLY) {console.log('');console.log('DRY-RUN complete. Re-run with --apply to commit changes.');+ console.log(`Would upload ${archives.length} archive files + manifest.`);+ console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`);return;}- // Step 5: Upload archives+ // ── Step 5: Upload archives ───────────────────────────────────────console.log('');console.log('Step 5: Uploading archives...');const uploadOps = archives.map(a => ({@@ -200,8 +256,6 @@ async function main() {encoding: 'base64',},});-- // Save invalid records if anyif (invalidRecords.length > 0) {uploadOps.push({key: 'file',@@ -216,17 +270,47 @@ async function main() {await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`);console.log(' Archives committed.');- // Step 6: Delete legacy files- console.log('Step 6: Deleting legacy files...');- const deleteOps = jsonlFiles.map(f => ({- key: 'deletedFile',- value: { path: f.path },- }));- await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`);- console.log(` Deleted ${jsonlFiles.length} files.`);+ // ── Step 6: Verify uploaded archives (re-download + checksum) ─────+ console.log('');+ console.log('Step 6: Verifying uploaded archives...');+ for (const archive of archives) {+ const downloaded = await downloadFile(archive.path);+ const remoteChecksum = await sha256(downloaded);+ if (remoteChecksum !== archive.checksum) {+ console.error(` CHECKSUM MISMATCH for ${archive.path}!`);+ console.error(` Expected: ${archive.checksum}`);+ console.error(` Got: ${remoteChecksum}`);+ console.error(' ABORTING — archives uploaded but legacy files NOT deleted.');+ console.error(' Investigate manually before retrying.');+ process.exit(1);+ }+ console.log(` ${archive.path}: checksum verified`);+ }+ console.log(' All archives verified.');++ // ── Step 7: Delete legacy files in batches ────────────────────────+ console.log('');+ console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`);+ const batches = [];+ for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) {+ batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE));+ }++ for (let i = 0; i < batches.length; i++) {+ const batch = batches[i];+ const deleteOps = batch.map(f => ({+ key: 'deletedFile',+ value: { path: f.path },+ }));+ await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`);+ console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`);+ }console.log('');console.log('DONE. Legacy telemetry compacted successfully.');+ console.log(` ${archives.length} monthly archives in archive/legacy/`);+ console.log(` ${sourceFileCount} legacy files removed`);+ console.log(` ${totalEvents} events preserved`);}main().catch(err => {