Skip to content

LIBERATED AI CHAT

TypeScript

fix: remove fake KV lock, add GET ?flush=force, add .gitignore

fabercebc27cf4c32
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6238342
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+node_modules/
+.next/
+.env
+.env.local
+.wrangler/
+dist/
+tsconfig.tsbuildinfo
diff --git a/functions/api/telemetry.ts b/functions/api/telemetry.ts
index 6de6052..9fc74ef 100644
--- a/functions/api/telemetry.ts
+++ b/functions/api/telemetry.ts
@@ -108,6 +108,25 @@ export const onRequestOptions: PagesFunction<Env> = async () => {
return new Response(null, { status: 204, headers: CORS_HEADERS })
}
+// GET ?flush=force — external cron trigger for guaranteed flush
+// Set up a CF Worker cron or external cron to hit this every 30 min
+// to ensure batches don't sit in KV forever during low-traffic periods.
+export const onRequestGet: PagesFunction<Env> = async (context) => {
+ const { request, env } = context
+ const url = new URL(request.url)
+
+ if (url.searchParams.get('flush') !== 'force') {
+ return jsonResponse({ status: 'ok', hint: 'POST events here, or GET ?flush=force to trigger a flush' }, 200)
+ }
+
+ if (!env.TELEMETRY_KV || !env.HF_TOKEN || !env.HF_DATASET_REPO) {
+ return jsonResponse({ error: 'KV or HF not configured' }, 503)
+ }
+
+ const flushed = await flushKVToHF(env)
+ return jsonResponse({ flushed }, 200)
+}
+
// Main handler
export const onRequestPost: PagesFunction<Env> = async (context) => {
const { request, env } = context
@@ -175,11 +194,11 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
}
if (shouldFlush) {
- // Acquire a simple lock to prevent concurrent flushes
- const lockValue = await env.TELEMETRY_KV.get('flush:lock')
- if (!lockValue) {
- context.waitUntil(flushKVToHF(env))
- }
+ // Flush is idempotent: concurrent flushes may produce duplicate
+ // JSONL files on HF, which is harmless (extra telemetry rows).
+ // KV is eventually consistent and can't do atomic locks, so we
+ // accept rare duplicates rather than pretend we can prevent them.
+ context.waitUntil(flushKVToHF(env))
}
return jsonResponse({
@@ -207,49 +226,45 @@ export const onRequestPost: PagesFunction<Env> = async (context) => {
// ── KV Flush: combine all pending batches into one HF commit ────────
-async function flushKVToHF(env: Env): Promise<void> {
+async function flushKVToHF(env: Env): Promise<number> {
const kv = env.TELEMETRY_KV!
+ const allKeys: { name: string }[] = []
- // Acquire lock (60s TTL — if flush crashes, lock auto-expires)
- await kv.put('flush:lock', '1', { expirationTtl: 60 })
-
- try {
- const allKeys: { name: string }[] = []
-
- let cursor: string | undefined
- do {
- const list = await kv.list({ prefix: 'batch:', cursor })
- allKeys.push(...list.keys)
- cursor = list.list_complete ? undefined : list.cursor
- } while (cursor)
+ let cursor: string | undefined
+ do {
+ const list = await kv.list({ prefix: 'batch:', cursor })
+ allKeys.push(...list.keys)
+ cursor = list.list_complete ? undefined : list.cursor
+ } while (cursor)
- if (allKeys.length === 0) return
+ if (allKeys.length === 0) return 0
- const chunks: string[] = []
- for (const key of allKeys) {
- const value = await kv.get(key.name)
- if (value) chunks.push(value)
- }
+ const chunks: string[] = []
+ for (const key of allKeys) {
+ const value = await kv.get(key.name)
+ if (value) chunks.push(value)
+ }
- if (chunks.length === 0) return
+ if (chunks.length === 0) return 0
- const combined = chunks.join('\n')
- const ts = new Date().toISOString().replace(/[:.]/g, '-')
- const hash = shortHash(combined)
- const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
- const branch = env.HF_DATASET_BRANCH || 'main'
+ const combined = chunks.join('\n')
+ const ts = new Date().toISOString().replace(/[:.]/g, '-')
+ const hash = shortHash(combined)
+ const filePath = `telemetry/batch_${ts}_${hash}.jsonl`
+ const branch = env.HF_DATASET_BRANCH || 'main'
- const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
+ const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
- if (ok) {
- await Promise.all(allKeys.map(key => kv.delete(key.name)))
- console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
- } else {
- console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
- }
- } finally {
- await kv.delete('flush:lock')
+ if (ok) {
+ // Delete flushed keys. If a concurrent flush already deleted some,
+ // the delete is a no-op — KV delete on missing keys is safe.
+ await Promise.all(allKeys.map(key => kv.delete(key.name)))
+ console.log(`[Telemetry] Flushed ${allKeys.length} batches → ${filePath}`)
+ return allKeys.length
}
+
+ console.error(`[Telemetry] Flush failed — ${allKeys.length} batches remain in KV`)
+ return 0
}
// ── HuggingFace Hub Commit ───────────────────────────────────────────