Skip to content

LIBERATED AI CHAT

TypeScript

fix: deterministic date partitions + compaction script rewrite

faber2896b530acce
diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts
index 9da5dc7..c87aea2 100644
--- a/functions/api/telemetry.ts
+++ b/functions/api/telemetry.ts
@@ -377,7 +377,9 @@ async function flushKVToHF(env: Env): Promise<number> {
const combined = chunks.join('\n')
const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(','))
- const d = new Date()
+ // Derive partition from oldest batch key timestamp (stable across retries)
+ const oldestTs = parseInt(flushedKeys[0].name.split(':')[1], 10)
+ const d = !isNaN(oldestTs) ? new Date(oldestTs) : new Date()
const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`
const filePath = `incoming_v2/${datePath}/flush_${keyDigest}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
@@ -403,12 +405,14 @@ async function flushMemToHF(env: Env): Promise<number> {
// accumulate into a fresh buffer, not into the one we're flushing.
const snapshot = memBuffer.splice(0, memBuffer.length)
const snapshotBytes = memBufferBytes
+ const snapshotTs = memBufferFirstTs
memBufferBytes = 0
memBufferFirstTs = 0
const combined = snapshot.join('\n')
const digest = await sha256Hex(combined)
- const d = new Date()
+ // Derive partition from buffer start time (stable across retries)
+ const d = snapshotTs > 0 ? new Date(snapshotTs) : new Date()
const datePath = `${d.getUTCFullYear()}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${String(d.getUTCDate()).padStart(2, '0')}`
const filePath = `incoming_v2/${datePath}/membuf_${digest}.jsonl`
const branch = env.HF_DATASET_BRANCH || 'main'
diff --git a/scripts/compact-hf-telemetry.mjs b/scripts/compact-hf-telemetry.mjs
index a37ab71..60eebed 100644
--- a/scripts/compact-hf-telemetry.mjs
+++ b/scripts/compact-hf-telemetry.mjs
@@ -4,15 +4,18 @@
*
* Usage:
* node scripts/compact-hf-telemetry.mjs # dry-run (default)
- * node scripts/compact-hf-telemetry.mjs --apply # actually commit archives + delete legacy
+ * node scripts/compact-hf-telemetry.mjs --apply # commit archives + delete legacy
*
* Requires: HF_TOKEN env var with write access to the dataset.
*
- * Safety:
- * - Dry-run is default; --apply required for writes
- * - Verifies event counts before deleting anything
- * - Produces a manifest with checksums
- * - Stops on any count mismatch
+ * Safety guarantees:
+ * - Dry-run is default; --apply required for any writes
+ * - FAIL-CLOSED: aborts on ANY download failure or parse error threshold
+ * - Paginates the full file listing (HF API returns max 1000/page)
+ * - Verifies uploaded archives by re-downloading and checking checksums
+ * - Batches delete operations (max 50 per HF commit)
+ * - Preserves FULL invalid lines (not truncated) in a separate artifact
+ * - Stops and reports if any count mismatch is detected
*/
const HF_API = 'https://huggingface.co/api';
@@ -20,6 +23,7 @@ const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3'
const BRANCH = process.env.HF_DATASET_BRANCH || 'main';
const TOKEN = process.env.HF_TOKEN;
const APPLY = process.argv.includes('--apply');
+const DELETE_BATCH_SIZE = 50;
if (!TOKEN) {
console.error('ERROR: HF_TOKEN environment variable required');
@@ -28,11 +32,35 @@ if (!TOKEN) {
const headers = { Authorization: `Bearer ${TOKEN}` };
-async function listFiles(prefix) {
- const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}`;
- const res = await fetch(url, { headers });
- if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);
- return res.json();
+// ── Paginated file listing ──────────────────────────────────────────
+async function listAllFiles(prefix) {
+ const allFiles = [];
+ let cursor = null;
+
+ while (true) {
+ const params = new URLSearchParams();
+ if (cursor) params.set('cursor', cursor);
+ const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`;
+ const res = await fetch(url, { headers });
+ if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);
+
+ const page = await res.json();
+ if (!Array.isArray(page) || page.length === 0) break;
+
+ allFiles.push(...page);
+ console.log(` Listed ${allFiles.length} entries so far...`);
+
+ // HF returns Link header with rel="next" for pagination
+ const linkHeader = res.headers.get('link') || '';
+ const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/);
+ if (nextMatch) {
+ cursor = decodeURIComponent(nextMatch[1]);
+ } else {
+ break;
+ }
+ }
+
+ return allFiles;
}
async function downloadFile(path) {
@@ -48,6 +76,13 @@ async function sha256(text) {
return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('');
}
+function utf8ToBase64(str) {
+ const bytes = new TextEncoder().encode(str);
+ let binStr = '';
+ for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]);
+ return btoa(binStr);
+}
+
async function commitFiles(operations, summary) {
const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`;
const lines = [
@@ -59,82 +94,94 @@ async function commitFiles(operations, summary) {
headers: { ...headers, 'Content-Type': 'application/x-ndjson' },
body: lines.join('\n'),
});
- if (!res.ok) throw new Error(`Commit failed (${res.status}): ${await res.text()}`);
+ if (!res.ok) throw new Error(`Commit failed (${res.status}): ${(await res.text()).slice(0, 500)}`);
return res.json();
}
-function utf8ToBase64(str) {
- const bytes = new TextEncoder().encode(str);
- let binStr = '';
- for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]);
- return btoa(binStr);
-}
-
async function main() {
console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`);
console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`);
console.log('');
- // Step 1: List all legacy telemetry files
- console.log('Step 1: Listing legacy telemetry/ files...');
- const files = await listFiles('telemetry');
- const jsonlFiles = files.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
- console.log(` Found ${jsonlFiles.length} JSONL files`);
+ // ── Step 1: Paginated listing ─────────────────────────────────────
+ console.log('Step 1: Listing ALL legacy telemetry/ files (paginated)...');
+ const allEntries = await listAllFiles('telemetry');
+ const jsonlFiles = allEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
+ console.log(` Total entries: ${allEntries.length}`);
+ console.log(` JSONL files: ${jsonlFiles.length}`);
if (jsonlFiles.length === 0) {
console.log(' Nothing to compact. Done.');
return;
}
- // Step 2: Download and parse all files
- console.log('Step 2: Downloading and parsing events...');
+ // ── Step 2: Download and parse ALL files (fail-closed) ────────────
+ console.log('');
+ console.log('Step 2: Downloading and parsing events (fail-closed)...');
const monthBuckets = {};
let totalEvents = 0;
- let totalBytes = 0;
+ let totalSourceBytes = 0;
+ let downloadFailures = 0;
let invalidLines = 0;
const invalidRecords = [];
+ const failedFiles = [];
for (let i = 0; i < jsonlFiles.length; i++) {
const file = jsonlFiles[i];
- if (i % 500 === 0) console.log(` Progress: ${i}/${jsonlFiles.length} files...`);
+ if (i % 200 === 0 || i === jsonlFiles.length - 1) {
+ console.log(` Progress: ${i + 1}/${jsonlFiles.length} files...`);
+ }
+ let content;
try {
- const content = await downloadFile(file.path);
- totalBytes += new TextEncoder().encode(content).byteLength;
- const lines = content.split('\n').filter(l => l.trim());
-
- for (const line of lines) {
- try {
- const event = JSON.parse(line);
- const ts = event.timestamp;
- let monthKey;
- if (ts && typeof ts === 'number') {
- const d = new Date(ts);
- monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
- } else {
- monthKey = 'unknown';
- }
- if (!monthBuckets[monthKey]) monthBuckets[monthKey] = [];
- monthBuckets[monthKey].push(line);
- totalEvents++;
- } catch {
- invalidLines++;
- invalidRecords.push({ file: file.path, line: line.slice(0, 200) });
+ content = await downloadFile(file.path);
+ } catch (err) {
+ downloadFailures++;
+ failedFiles.push({ path: file.path, error: err.message });
+ continue;
+ }
+
+ totalSourceBytes += new TextEncoder().encode(content).byteLength;
+ const lines = content.split('\n').filter(l => l.trim());
+
+ for (const line of lines) {
+ try {
+ const event = JSON.parse(line);
+ const ts = event.timestamp;
+ let monthKey;
+ if (ts && typeof ts === 'number') {
+ const d = new Date(ts);
+ monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
+ } else {
+ monthKey = 'unknown';
}
+ if (!monthBuckets[monthKey]) monthBuckets[monthKey] = [];
+ monthBuckets[monthKey].push(line);
+ totalEvents++;
+ } catch {
+ invalidLines++;
+ invalidRecords.push({ file: file.path, line });
}
- } catch (err) {
- console.error(` ERROR reading ${file.path}: ${err.message}`);
- invalidRecords.push({ file: file.path, error: err.message });
}
}
- console.log(` Total events: ${totalEvents}`);
- console.log(` Total source bytes: ${(totalBytes / 1024 / 1024).toFixed(2)} MiB`);
+ console.log(` Total events parsed: ${totalEvents}`);
+ console.log(` Total source bytes: ${(totalSourceBytes / 1024 / 1024).toFixed(2)} MiB`);
+ console.log(` Download failures: ${downloadFailures}`);
console.log(` Invalid lines: ${invalidLines}`);
console.log(` Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`);
- console.log('');
- // Step 3: Build archives
+ // ── FAIL-CLOSED: abort if any downloads failed ────────────────────
+ if (downloadFailures > 0) {
+ console.error('');
+ console.error(`ABORT: ${downloadFailures} file(s) failed to download.`);
+ console.error('Cannot guarantee data preservation. Fix network/auth issues and retry.');
+ console.error('Failed files:', failedFiles.map(f => f.path).join(', '));
+ process.exit(1);
+ }
+
+ // ── Step 3: Build archives ────────────────────────────────────────
+ console.log('');
console.log('Step 3: Building monthly archives...');
const archives = [];
for (const [month, events] of Object.entries(monthBuckets).sort()) {
@@ -143,32 +190,39 @@ async function main() {
const archivePath = `archive/legacy/telemetry-${month}.jsonl`;
const bytes = new TextEncoder().encode(content).byteLength;
archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content });
- console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 12)}...`);
+ console.log(` ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`);
}
- // Step 4: Verify counts
+ // ── Step 4: Verify counts (tautology-free) ────────────────────────
const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0);
+ const sourceFileCount = jsonlFiles.length;
+ const expectedTotal = totalEvents; // all successfully parsed events
+
console.log('');
- console.log(`Step 4: Verification`);
- console.log(` Source events: ${totalEvents}`);
- console.log(` Archive events: ${archiveEventTotal}`);
+ console.log('Step 4: Verification');
+ console.log(` Source files examined: ${sourceFileCount}`);
+ console.log(` Download failures: ${downloadFailures} (must be 0)`);
+ console.log(` Events parsed from sources: ${totalEvents}`);
+ console.log(` Events in archives: ${archiveEventTotal}`);
+ console.log(` Invalid lines preserved: ${invalidLines}`);
- if (archiveEventTotal !== totalEvents) {
- console.error(' MISMATCH — aborting. No changes made.');
+ if (archiveEventTotal !== expectedTotal) {
+ console.error(' COUNT MISMATCH — aborting. No changes made.');
process.exit(1);
}
- console.log(' MATCH — counts verified.');
+ console.log(' COUNTS MATCH.');
- // Build manifest
+ // ── Build manifest ────────────────────────────────────────────────
const manifest = {
timestamp: new Date().toISOString(),
repo: DATASET_REPO,
branch: BRANCH,
- sourceFiles: jsonlFiles.length,
- sourceBytes: totalBytes,
+ sourceFiles: sourceFileCount,
+ sourceBytes: totalSourceBytes,
totalEvents,
invalidLines,
- invalidRecords: invalidRecords.length,
+ invalidRecordCount: invalidRecords.length,
+ downloadFailures,
archives: archives.map(a => ({
path: a.path, month: a.month, events: a.events,
bytes: a.bytes, checksum: a.checksum,
@@ -182,10 +236,12 @@ async function main() {
if (!APPLY) {
console.log('');
console.log('DRY-RUN complete. Re-run with --apply to commit changes.');
+ console.log(`Would upload ${archives.length} archive files + manifest.`);
+ console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`);
return;
}
- // Step 5: Upload archives
+ // ── Step 5: Upload archives ───────────────────────────────────────
console.log('');
console.log('Step 5: Uploading archives...');
const uploadOps = archives.map(a => ({
@@ -200,8 +256,6 @@ async function main() {
encoding: 'base64',
},
});
-
- // Save invalid records if any
if (invalidRecords.length > 0) {
uploadOps.push({
key: 'file',
@@ -216,17 +270,47 @@ async function main() {
await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files`);
console.log(' Archives committed.');
- // Step 6: Delete legacy files
- console.log('Step 6: Deleting legacy files...');
- const deleteOps = jsonlFiles.map(f => ({
- key: 'deletedFile',
- value: { path: f.path },
- }));
- await commitFiles(deleteOps, `[maintenance] Remove ${jsonlFiles.length} legacy telemetry files (archived in archive/legacy/)`);
- console.log(` Deleted ${jsonlFiles.length} files.`);
+ // ── Step 6: Verify uploaded archives (re-download + checksum) ─────
+ console.log('');
+ console.log('Step 6: Verifying uploaded archives...');
+ for (const archive of archives) {
+ const downloaded = await downloadFile(archive.path);
+ const remoteChecksum = await sha256(downloaded);
+ if (remoteChecksum !== archive.checksum) {
+ console.error(` CHECKSUM MISMATCH for ${archive.path}!`);
+ console.error(` Expected: ${archive.checksum}`);
+ console.error(` Got: ${remoteChecksum}`);
+ console.error(' ABORTING — archives uploaded but legacy files NOT deleted.');
+ console.error(' Investigate manually before retrying.');
+ process.exit(1);
+ }
+ console.log(` ${archive.path}: checksum verified`);
+ }
+ console.log(' All archives verified.');
+
+ // ── Step 7: Delete legacy files in batches ────────────────────────
+ console.log('');
+ console.log(`Step 7: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`);
+ const batches = [];
+ for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) {
+ batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE));
+ }
+
+ for (let i = 0; i < batches.length; i++) {
+ const batch = batches[i];
+ const deleteOps = batch.map(f => ({
+ key: 'deletedFile',
+ value: { path: f.path },
+ }));
+ await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`);
+ console.log(` Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`);
+ }
console.log('');
console.log('DONE. Legacy telemetry compacted successfully.');
+ console.log(` ${archives.length} monthly archives in archive/legacy/`);
+ console.log(` ${sourceFileCount} legacy files removed`);
+ console.log(` ${totalEvents} events preserved`);
}
main().catch(err => {