AI-native design editor. Open-source Figma alternative.
feat(feedback): record daily metrics snapshots (#1316)
Danila Poyarkova5efcd5b3bbb
diff --git a/bun.lock b/bun.lockindex 324c50911..1d7311776 100644--- a/bun.lock+++ b/bun.lock@@ -416,6 +416,7 @@"name": "@open-pencil/feedback","version": "0.16.0","dependencies": {+ "@internationalized/date": "^3.12.3","@open-pencil/scene-graph": "workspace:*","@vue-email/components": "^0.0.21","@vue-email/render": "^0.0.9",diff --git a/oxlint.json b/oxlint.jsonindex ae31a1c65..4cee0bcfa 100644--- a/oxlint.json+++ b/oxlint.json@@ -452,10 +452,16 @@{"files": ["packages/cloud/src/server/**/*.ts"],"rules": {- "open-pencil/no-cloud-raw-sql": "error",+ "open-pencil/no-raw-sql": "error","open-pencil/no-cloud-dynamic-html": "error"}},+ {+ "files": ["packages/feedback/src/**/*.ts"],+ "rules": {+ "open-pencil/no-raw-sql": "error"+ }+ },{"files": ["packages/cloud/src/server/db/expressions.ts",@@ -463,7 +469,7 @@"packages/cloud/src/server/db/migrations/**/*.ts"],"rules": {- "open-pencil/no-cloud-raw-sql": "off"+ "open-pencil/no-raw-sql": "off"}},{diff --git a/packages/feedback/.dev.vars.example b/packages/feedback/.dev.vars.exampleindex 0c0d0caa5..c6b4cff2a 100644--- a/packages/feedback/.dev.vars.example+++ b/packages/feedback/.dev.vars.example@@ -2,3 +2,6 @@ POSTHOG_KEY=phc_development# Cloudflare's Turnstile test secret, which accepts every token and reports example.com.TURNSTILE_SECRET=1x0000000000000000000000000000000AATURNSTILE_HOSTNAMES=example.com+# The metrics snapshot's tokens; without real ones its requests fail and nothing is recorded.+GITHUB_TOKEN=github_development+CLOUDFLARE_ANALYTICS_TOKEN=cloudflare_developmentdiff --git a/packages/feedback/AGENTS.md b/packages/feedback/AGENTS.mdindex 663e55f88..c9805b4fb 100644--- a/packages/feedback/AGENTS.md+++ b/packages/feedback/AGENTS.md@@ -4,7 +4,9 @@ Cloudflare Worker for pings, opt-in usage stats, and the Cloud waitlist; overvie- Every field the app may send is a closed Valibot list in `src/contract/schema.ts`, with its plain-language meaning in `src/contract/describe.ts`; the privacy page renders both. Add an event or property in both files and update `packages/docs/overview/privacy.md` prose if what is collected changes (`tests/contract.test.ts`).- The Worker never stores or forwards a client address: rate limits key on it in memory, PostHog requests are new requests from the Worker, and Turnstile is verified without `remoteip` (`tests/ingest.test.ts`).-- Schema changes are new files in `migrations/`, applied by `wrangler d1 migrations`; keep `src/db/schema.ts` in step (`tests/harness.ts` applies them).+- Schema changes are new files in `migrations/`, applied by `wrangler d1 migrations`; keep `src/db/schema.ts` in step (`tests/harness.ts` applies them). Everything else, tests included, queries D1 through the typed Kysely database; raw `sql` templates are rejected (`open-pencil/no-raw-sql`).+- Calendar days are `CalendarDate`s from `@internationalized/date` (`src/days.ts`), turned into ISO dates only where they are stored or sent; `Date` is for moments such as the clock and sign-up times (`tests/snapshot.test.ts`).- Routes are chained Hono routers that validate bodies with `validatedJSON(schema)` and export `FeedbackAPI`; clients use `createFeedbackClient` (`hc<FeedbackAPI>`), as Cloud's do, never hand-written paths (`tests/harness.ts` uses `testClient`).- Bindings are declared in `src/env.ts` with importable Workers types, so the app can typecheck `FeedbackAPI`; never use the generated global `Env` in code the router reaches. Run `bun --filter @open-pencil/feedback types` after changing `wrangler.jsonc`; `src/index.ts` checks `Env` against `FeedbackBindings` and `bun run test:feedback` checks the generated file.- Outside services (`fetch`, the clock) come through `FeedbackServices`, and bindings through `Env`, so tests run the real app against a local D1 under `bun test` (`tests/harness.ts`).+- The metrics snapshot (`src/snapshot/`) keys every table on the UTC day it describes and replaces whole days in one D1 batch (`replaceDays`), so a rerun corrects a day and a failed write keeps the old rows; it writes the table it resumes from (`site_visits`) last, runs each source on its own, and parses every answer with Valibot before writing (`tests/snapshot.test.ts`).diff --git a/packages/feedback/README.md b/packages/feedback/README.mdindex 471880da4..21e2b9716 100644--- a/packages/feedback/README.md+++ b/packages/feedback/README.md@@ -8,6 +8,8 @@ The Cloudflare Worker behind `feedback.openpencil.dev`. It is operated by the Op| `POST /ingest` | Opt-in usage stats, forwarded to PostHog EU. Refuses any event or property not in `src/contract`. | Nothing. PostHog gets no client address, person profile, or GeoIP. || `POST /waitlist` | OpenPencil Cloud waitlist sign-ups from the website and the web app's Settings, checked by Turnstile and confirmed by email. | The address and optional note; unconfirmed sign-ups are removed after seven days. |+Two daily cron jobs run besides the routes. Just after midnight UTC the metrics snapshot records the day that ended for the team's metrics dashboard: GitHub stars and forks, running download totals per release and kind of file (installers, macOS updates, and `latest.json` update checks), repository views, clones, and referrers, and Cloudflare's visits to `openpencil.dev` and `app.openpencil.dev` with the website's top pages and countries. GitHub keeps traffic for 14 days and Cloudflare's Free plan keeps visits for 31, so these tables are the only lasting history; each run also fetches again the last few days Cloudflare may still have been counting, and fills any missing days it still can. At 03:17 UTC, retention deletes unconfirmed waitlist sign-ups.+`@open-pencil/feedback/contract` holds the request schemas and the event list, and `@open-pencil/feedback/client` is a client typed by the Worker's routes (`hc<FeedbackAPI>`). The app sends through that client and the privacy page renders its tables from the contract, so the documented, sent, and accepted data cannot drift apart.## Development@@ -29,3 +31,4 @@ Done once on the Cloudflare account; `.github/workflows/feedback.yml` deploys ev2. Onboard `openpencil.dev` to Email Service so `[email protected]` can send. Sending to addresses outside the account needs the Workers Paid plan; send one confirmation to an outside address before announcing the waitlist.3. Create a Turnstile widget for `openpencil.dev` and `app.openpencil.dev`, the hostnames in `TURNSTILE_HOSTNAMES` (its public site key is `TURNSTILE_SITE_KEY` in the contract; the form gives it the `waitlist` action), then `bunx wrangler secret put TURNSTILE_SECRET`.4. Create a PostHog EU project with "Discard client IP data" on and data retention at 12 months, as the privacy page promises, then `bunx wrangler secret put POSTHOG_KEY` with its project key.+5. For the metrics snapshot, `bunx wrangler secret put GITHUB_TOKEN` with a fine-grained token for the repository in `GITHUB_REPOSITORY` that has read-only **Administration** access, which the traffic API requires (every token can already read the public repository's releases), and `bunx wrangler secret put CLOUDFLARE_ANALYTICS_TOKEN` with an account API token that has **Zone → Analytics → Read** on the zone in `SITE_ZONE_ID`.diff --git a/packages/feedback/migrations/0003_metrics_snapshots.sql b/packages/feedback/migrations/0003_metrics_snapshots.sqlnew file mode 100644index 000000000..7751002f1--- /dev/null+++ b/packages/feedback/migrations/0003_metrics_snapshots.sql@@ -0,0 +1,63 @@+-- Daily snapshots for the metrics dashboard. GitHub only reports running totals and Cloudflare+-- keeps 31 days on the Free plan, so the history exists only if it is recorded each day. `day`+-- is the UTC day a row describes; totals are as of the end of that day.++-- Stars and forks at the end of the day.+CREATE TABLE github_stars (+ day TEXT PRIMARY KEY,+ stars INTEGER NOT NULL,+ forks INTEGER NOT NULL+) WITHOUT ROWID;++-- Running download totals per release and kind of file; the daily change is the difference+-- between two days.+CREATE TABLE release_downloads (+ day TEXT NOT NULL,+ tag TEXT NOT NULL,+ kind TEXT NOT NULL,+ downloads INTEGER NOT NULL,+ PRIMARY KEY (day, tag, kind)+) WITHOUT ROWID;++-- Repository views and clones, which GitHub keeps for 14 days.+CREATE TABLE repo_traffic (+ day TEXT PRIMARY KEY,+ views INTEGER NOT NULL,+ unique_views INTEGER NOT NULL,+ clones INTEGER NOT NULL,+ unique_clones INTEGER NOT NULL+) WITHOUT ROWID;++-- The sites sending visitors to the repository over the 14 days before `day`, as GitHub reports+-- them; it has no per-day breakdown.+CREATE TABLE repo_referrers (+ day TEXT NOT NULL,+ referrer TEXT NOT NULL,+ views INTEGER NOT NULL,+ unique_views INTEGER NOT NULL,+ PRIMARY KEY (day, referrer)+) WITHOUT ROWID;++-- Cloudflare's visits per hostname: page views that arrived from another site or a link.+CREATE TABLE site_visits (+ day TEXT NOT NULL,+ host TEXT NOT NULL,+ visits INTEGER NOT NULL,+ PRIMARY KEY (day, host)+) WITHOUT ROWID;++-- The website's most visited pages each day.+CREATE TABLE site_pages (+ day TEXT NOT NULL,+ path TEXT NOT NULL,+ visits INTEGER NOT NULL,+ PRIMARY KEY (day, path)+) WITHOUT ROWID;++-- The website's visits by country each day, as two-letter codes.+CREATE TABLE site_countries (+ day TEXT NOT NULL,+ country TEXT NOT NULL,+ visits INTEGER NOT NULL,+ PRIMARY KEY (day, country)+) WITHOUT ROWID;diff --git a/packages/feedback/package.json b/packages/feedback/package.jsonindex 66291aec0..691846e9b 100644--- a/packages/feedback/package.json+++ b/packages/feedback/package.json@@ -3,7 +3,7 @@"version": "0.16.0","private": true,"license": "MIT",- "description": "The OpenPencil feedback service on Cloudflare Workers: usage pings, opt-in usage stats, and the Cloud waitlist.",+ "description": "The OpenPencil feedback service on Cloudflare Workers: usage pings, opt-in usage stats, the Cloud waitlist, and the daily metrics snapshot.","type": "module","imports": {"#feedback/*": "./src/*",@@ -22,6 +22,7 @@"check:types": "wrangler types --include-runtime false --check"},"dependencies": {+ "@internationalized/date": "^3.12.3","@open-pencil/scene-graph": "workspace:*","@vue-email/components": "^0.0.21","@vue-email/render": "^0.0.9",diff --git a/packages/feedback/src/app.ts b/packages/feedback/src/app.tsindex b8fe7dd4c..5ba6556a6 100644--- a/packages/feedback/src/app.ts+++ b/packages/feedback/src/app.ts@@ -11,7 +11,8 @@ export type { FeedbackBindings, FeedbackEnv, FeedbackServices } from './env'const MAX_BODY_BYTES = 16 * 1024-const defaultServices: FeedbackServices = {+/** The real outside world; tests pass their own. */+export const defaultServices: FeedbackServices = {fetch: (input, init) => fetch(input, init),now: () => new Date()}diff --git a/packages/feedback/src/days.ts b/packages/feedback/src/days.tsnew file mode 100644index 000000000..1bf3d58ba--- /dev/null+++ b/packages/feedback/src/days.ts@@ -0,0 +1,21 @@+import { type CalendarDate, fromDate, parseAbsolute, toCalendarDate } from '@internationalized/date'++/**+ * Pings, snapshots, and the dashboard count in UTC calendar days. Days are `CalendarDate`s in+ * code and ISO dates (`2026-10-19`) only where they are stored or sent.+ */+const UTC = 'UTC'++/** The UTC day a moment falls on. */+export const utcDayOf = (moment: Date): CalendarDate => toCalendarDate(fromDate(moment, UTC))++/** The UTC day of an ISO timestamp such as GitHub's `2026-10-19T00:00:00Z`. */+export const utcDayOfTimestamp = (timestamp: string): CalendarDate =>+ toCalendarDate(parseAbsolute(timestamp, UTC))++/** Every day from `start` to `end`, both included, oldest first. */+export function eachDay(start: CalendarDate, end: CalendarDate): CalendarDate[] {+ const days: CalendarDate[] = []+ for (let day = start; day.compare(end) <= 0; day = day.add({ days: 1 })) days.push(day)+ return days+}diff --git a/packages/feedback/src/db/schema.ts b/packages/feedback/src/db/schema.tsindex 60fa3674e..57e2c8b36 100644--- a/packages/feedback/src/db/schema.ts+++ b/packages/feedback/src/db/schema.ts@@ -22,8 +22,61 @@ export interface WaitlistTable {confirmedAt: string | null}+export interface GithubStarsTable {+ day: string+ stars: number+ forks: number+}++export interface ReleaseDownloadsTable {+ day: string+ tag: string+ kind: string+ downloads: number+}++export interface RepoTrafficTable {+ day: string+ views: number+ uniqueViews: number+ clones: number+ uniqueClones: number+}++export interface RepoReferrersTable {+ day: string+ referrer: string+ views: number+ uniqueViews: number+}++export interface SiteVisitsTable {+ day: string+ host: string+ visits: number+}++export interface SitePagesTable {+ day: string+ path: string+ visits: number+}++export interface SiteCountriesTable {+ day: string+ country: string+ visits: number+}+/** The D1 tables `migrations/` creates, in Kysely's camelCase. */export interface FeedbackDatabase {usagePings: UsagePingTablewaitlist: WaitlistTable+ githubStars: GithubStarsTable+ releaseDownloads: ReleaseDownloadsTable+ repoTraffic: RepoTrafficTable+ repoReferrers: RepoReferrersTable+ siteVisits: SiteVisitsTable+ sitePages: SitePagesTable+ siteCountries: SiteCountriesTable}diff --git a/packages/feedback/src/env.ts b/packages/feedback/src/env.tsindex af3abc03d..e60e7abe1 100644--- a/packages/feedback/src/env.ts+++ b/packages/feedback/src/env.ts@@ -18,6 +18,16 @@ export interface FeedbackBindings {/** The sites whose Turnstile passes a sign-up accepts, separated by commas. */TURNSTILE_HOSTNAMES: stringTURNSTILE_SECRET: string+ /** `owner/name` of the repository whose stars, downloads, and traffic are recorded. */+ GITHUB_REPOSITORY: string+ /** A fine-grained token that can read the repository's traffic (Administration: read). */+ GITHUB_TOKEN: string+ /** The Cloudflare zone holding `SITE_HOSTS`, for its visit analytics. */+ SITE_ZONE_ID: string+ /** The hostnames whose visits are recorded, separated by commas; the first is the website. */+ SITE_HOSTS: string+ /** A Cloudflare API token that can read the zone's analytics. */+ CLOUDFLARE_ANALYTICS_TOKEN: string}export type FeedbackEnv = { Bindings: FeedbackBindings }diff --git a/packages/feedback/src/index.ts b/packages/feedback/src/index.tsindex 52d2e75c5..b9f9416c4 100644--- a/packages/feedback/src/index.ts+++ b/packages/feedback/src/index.ts@@ -1,13 +1,20 @@-import { createFeedbackApp } from './app'+import { createFeedbackApp, defaultServices } from './app'import type { FeedbackBindings } from './env'import { runRetention } from './retention'+import { runSnapshot } from './snapshot'const app = createFeedbackApp()+/** The metrics snapshot's trigger in `wrangler.jsonc`; the other one runs retention. */+const SNAPSHOT_CRON = '5 0 * * *'export default {// `satisfies` keeps the declared bindings in step with `wrangler types`' generated `Env`.fetch: (request, env, ctx) => app.fetch(request, env satisfies FeedbackBindings, ctx),- scheduled(_controller, env, ctx) {- ctx.waitUntil(runRetention(env, new Date()))+ scheduled(controller, env, ctx) {+ ctx.waitUntil(+ controller.cron === SNAPSHOT_CRON+ ? runSnapshot(env, defaultServices)+ : runRetention(env, new Date())+ )}} satisfies ExportedHandler<Env>diff --git a/packages/feedback/src/ping/record.ts b/packages/feedback/src/ping/record.tsindex e6044c39c..02b05cdbf 100644--- a/packages/feedback/src/ping/record.ts+++ b/packages/feedback/src/ping/record.ts@@ -1,4 +1,5 @@import type { Ping } from '#feedback/contract'+import { utcDayOf } from '#feedback/days'import type { FeedbackDatabase } from '#feedback/db'import type { Kysely } from 'kysely'@@ -10,7 +11,7 @@ export async function recordPing(): Promise<void> {await db.insertInto('usagePings')- .values({ ...ping, day: now.toISOString().slice(0, 10), count: 1 })+ .values({ ...ping, day: utcDayOf(now).toString(), count: 1 }).onConflict((conflict) =>conflict.columns(['day', 'version', 'platform', 'arch', 'app', 'browser', 'usageStats'])diff --git a/packages/feedback/src/snapshot/assets.ts b/packages/feedback/src/snapshot/assets.tsnew file mode 100644index 000000000..9b86374ce--- /dev/null+++ b/packages/feedback/src/snapshot/assets.ts@@ -0,0 +1,27 @@+/**+ * What a release file is, from the names our release workflow gives them. Installers on+ * Windows and Linux double as their update packages; on macOS updates are their own archive, and+ * every running desktop app fetches `latest.json` when it checks for an update. Signatures,+ * checksums, and manifests are not counted.+ */+export type ReleaseFileKind =+ | 'macos-installer'+ | 'macos-update'+ | 'windows-installer'+ | 'linux-installer'+ | 'update-check'++const KINDS: [suffix: string, kind: ReleaseFileKind][] = [+ ['.app.tar.gz', 'macos-update'],+ ['.dmg', 'macos-installer'],+ ['.exe', 'windows-installer'],+ ['.msi', 'windows-installer'],+ ['.deb', 'linux-installer'],+ ['.rpm', 'linux-installer'],+ ['.AppImage', 'linux-installer']+]++export function releaseFileKind(name: string): ReleaseFileKind | undefined {+ if (name === 'latest.json') return 'update-check'+ return KINDS.find(([suffix]) => name.endsWith(suffix))?.[1]+}diff --git a/packages/feedback/src/snapshot/cloudflare.ts b/packages/feedback/src/snapshot/cloudflare.tsnew file mode 100644index 000000000..103e93e6f--- /dev/null+++ b/packages/feedback/src/snapshot/cloudflare.ts@@ -0,0 +1,127 @@+import type { FeedbackServices } from '#feedback/env'+import { type CalendarDate, parseDate } from '@internationalized/date'+import * as v from 'valibot'++const GRAPHQL_URL = 'https://api.cloudflare.com/client/v4/graphql'+const REQUEST_TIMEOUT_MS = 15_000+/** How many of the website's pages a day keeps; the long tail is one-off paths and scanners. */+export const TOP_PAGES = 50+/** More than there are countries, so a day keeps all of them. */+const COUNTRIES = 300++// Visits from people's browsers only: `eyeball` leaves out Cloudflare's own Workers and checks.+const VISITS_QUERY = `query ($zone: String!, $start: Date!, $end: Date!, $hosts: [String!]) {+ viewer { zones(filter: { zoneTag: $zone }) {+ httpRequestsAdaptiveGroups(+ limit: 10000+ filter: { date_geq: $start, date_leq: $end, requestSource: "eyeball", clientRequestHTTPHost_in: $hosts }+ ) { sum { visits } dimensions { date clientRequestHTTPHost } }+ } }+}`++const BREAKDOWN_QUERY = `query ($zone: String!, $day: Date!, $host: String!, $pages: Int!, $countries: Int!) {+ viewer { zones(filter: { zoneTag: $zone }) {+ pages: httpRequestsAdaptiveGroups(+ limit: $pages+ orderBy: [sum_visits_DESC]+ filter: { date: $day, requestSource: "eyeball", clientRequestHTTPHost: $host }+ ) { sum { visits } dimensions { clientRequestPath } }+ countries: httpRequestsAdaptiveGroups(+ limit: $countries+ orderBy: [sum_visits_DESC]+ filter: { date: $day, requestSource: "eyeball", clientRequestHTTPHost: $host }+ ) { sum { visits } dimensions { clientCountryName } }+ } }+}`++const Visits = v.object({ visits: v.number() })+const answer = <T extends v.GenericSchema>(zone: T) =>+ v.object({+ data: v.nullable(v.object({ viewer: v.object({ zones: v.array(zone) }) })),+ errors: v.nullish(v.array(v.object({ message: v.string() })))+ })+const VisitsZone = v.object({+ httpRequestsAdaptiveGroups: v.array(+ v.object({+ sum: Visits,+ dimensions: v.object({ date: v.string(), clientRequestHTTPHost: v.string() })+ })+ )+})+const BreakdownZone = v.object({+ pages: v.array(+ v.object({ sum: Visits, dimensions: v.object({ clientRequestPath: v.string() }) })+ ),+ countries: v.array(+ v.object({ sum: Visits, dimensions: v.object({ clientCountryName: v.string() }) })+ )+})++/** Cloudflare's GraphQL analytics for one zone; a failed or unexpected answer throws. */+export function cloudflareClient(fetcher: FeedbackServices['fetch'], zone: string, token: string) {+ async function query<T extends v.GenericSchema>(+ zoneSchema: T,+ text: string,+ variables: Record<string, unknown>+ ): Promise<v.InferOutput<T>> {+ const schema = answer(zoneSchema)+ const response = await fetcher(GRAPHQL_URL, {+ method: 'POST',+ headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },+ body: JSON.stringify({ query: text, variables: { zone, ...variables } }),+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)+ })+ if (!response.ok) throw new Error(`Cloudflare analytics answered ${response.status}`)+ const result = v.safeParse(v.pipe(v.string(), v.parseJson(), schema), await response.text())+ if (!result.success) throw new Error('Cloudflare analytics sent an unexpected answer')+ const first = result.output.errors?.at(0)+ if (first) throw new Error(`Cloudflare analytics refused the query: ${first.message}`)+ const [zoneData] = result.output.data?.viewer.zones ?? []+ if (!zoneData) throw new Error('Cloudflare analytics returned no data for the zone')+ return zoneData+ }++ return {+ /** Visits per day and hostname from `start` to `end`, both included. */+ async visits(start: CalendarDate, end: CalendarDate, hosts: string[]) {+ const zoneData = await query(VisitsZone, VISITS_QUERY, {+ start: start.toString(),+ end: end.toString(),+ hosts+ })+ return zoneData.httpRequestsAdaptiveGroups.map((group) => ({+ // Parsed rather than copied, so a malformed date fails the run instead of being stored.+ day: parseDate(group.dimensions.date).toString(),+ host: group.dimensions.clientRequestHTTPHost,+ visits: group.sum.visits+ }))+ },++ /** One day of a host's most visited pages and its visits by country. */+ async breakdown(date: CalendarDate, host: string) {+ const day = date.toString()+ const zoneData = await query(BreakdownZone, BREAKDOWN_QUERY, {+ day,+ host,+ pages: TOP_PAGES,+ countries: COUNTRIES+ })+ return {+ pages: zoneData.pages+ .filter((group) => group.sum.visits > 0)+ .map((group) => ({+ day,+ path: group.dimensions.clientRequestPath,+ visits: group.sum.visits+ })),+ countries: zoneData.countries+ .filter((group) => group.sum.visits > 0)+ .map((group) => ({+ day,+ country: group.dimensions.clientCountryName,+ visits: group.sum.visits+ }))+ }+ }+ }+}diff --git a/packages/feedback/src/snapshot/github.ts b/packages/feedback/src/snapshot/github.tsnew file mode 100644index 000000000..4ef792cc9--- /dev/null+++ b/packages/feedback/src/snapshot/github.ts@@ -0,0 +1,140 @@+import { utcDayOfTimestamp } from '#feedback/days'+import type { FeedbackServices } from '#feedback/env'+import type { CalendarDate } from '@internationalized/date'+import { groupBy, sumBy } from 'es-toolkit'+import * as v from 'valibot'++import { releaseFileKind } from './assets'++const API = 'https://api.github.com'+const REQUEST_TIMEOUT_MS = 15_000+// GitHub's maximum page size; a repository with more releases takes several pages.+const RELEASES_PER_PAGE = 100++const RepositorySchema = v.object({ stargazers_count: v.number(), forks_count: v.number() })+const ReleasesSchema = v.array(+ v.object({+ tag_name: v.string(),+ assets: v.array(v.object({ name: v.string(), download_count: v.number() }))+ })+)+const TrafficDaySchema = v.object({ timestamp: v.string(), count: v.number(), uniques: v.number() })+const ViewsSchema = v.object({ views: v.array(TrafficDaySchema) })+const ClonesSchema = v.object({ clones: v.array(TrafficDaySchema) })+const ReferrersSchema = v.array(+ v.object({ referrer: v.string(), count: v.number(), uniques: v.number() })+)++export interface GithubTotals {+ stars: number+ forks: number+ /** Running download totals per release and kind of file. */+ downloads: { tag: string; kind: string; downloads: number }[]+}++export interface GithubTraffic {+ days: {+ day: CalendarDate+ views: number+ uniqueViews: number+ clones: number+ uniqueClones: number+ }[]+ referrers: { referrer: string; views: number; uniqueViews: number }[]+}++/** Reads the repository's GitHub API with the snapshot's token; a failed or unexpected answer throws. */+export function githubClient(+ fetcher: FeedbackServices['fetch'],+ repository: string,+ token: string+) {+ async function get<T extends v.GenericSchema>(+ path: string,+ schema: T+ ): Promise<v.InferOutput<T>> {+ const response = await fetcher(`${API}/repos/${repository}${path}`, {+ headers: {+ Accept: 'application/vnd.github+json',+ Authorization: `Bearer ${token}`,+ 'User-Agent': 'openpencil-feedback',+ 'X-GitHub-Api-Version': '2026-03-10'+ },+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)+ })+ if (!response.ok)+ throw new Error(`GitHub /repos/${repository}${path} answered ${response.status}`)+ const result = v.safeParse(v.pipe(v.string(), v.parseJson(), schema), await response.text())+ if (!result.success)+ throw new Error(`GitHub /repos/${repository}${path} sent an unexpected answer`)+ return result.output+ }++ async function releases() {+ const all: v.InferOutput<typeof ReleasesSchema> = []+ for (let page = 1; ; page++) {+ const batch = await get(+ `/releases?per_page=${RELEASES_PER_PAGE}&page=${page}`,+ ReleasesSchema+ )+ all.push(...batch)+ if (batch.length < RELEASES_PER_PAGE) return all+ }+ }++ return {+ async totals(): Promise<GithubTotals> {+ const [repository, published] = await Promise.all([get('', RepositorySchema), releases()])+ const files = published.flatMap((release) =>+ release.assets.flatMap((asset) => {+ const kind = releaseFileKind(asset.name)+ return kind ? [{ tag: release.tag_name, kind, downloads: asset.download_count }] : []+ })+ )+ const downloads = Object.values(+ groupBy(files, (file) => `${file.tag}\n${file.kind}`)+ ).flatMap((group) =>+ group[0]+ ? [+ {+ tag: group[0].tag,+ kind: group[0].kind,+ downloads: sumBy(group, (file) => file.downloads)+ }+ ]+ : []+ )+ return { stars: repository.stargazers_count, forks: repository.forks_count, downloads }+ },++ /** The last 14 days of views and clones, and the referrers over that window. */+ async traffic(): Promise<GithubTraffic> {+ const [views, clones, referrers] = await Promise.all([+ get('/traffic/views?per=day', ViewsSchema),+ get('/traffic/clones?per=day', ClonesSchema),+ get('/traffic/popular/referrers', ReferrersSchema)+ ])+ // GitHub stamps each day with its UTC midnight; days are matched by their date.+ const dayOf = (entry: { timestamp: string }) => utcDayOfTimestamp(entry.timestamp)+ const clonesByDay = new Map(clones.clones.map((entry) => [dayOf(entry).toString(), entry]))+ return {+ days: views.views.map((entry) => {+ const day = dayOf(entry)+ const cloned = clonesByDay.get(day.toString())+ return {+ day,+ views: entry.count,+ uniqueViews: entry.uniques,+ clones: cloned?.count ?? 0,+ uniqueClones: cloned?.uniques ?? 0+ }+ }),+ referrers: referrers.map((referrer) => ({+ referrer: referrer.referrer,+ views: referrer.count,+ uniqueViews: referrer.uniques+ }))+ }+ }+ }+}diff --git a/packages/feedback/src/snapshot/index.ts b/packages/feedback/src/snapshot/index.tsnew file mode 100644index 000000000..2baff4efb--- /dev/null+++ b/packages/feedback/src/snapshot/index.ts@@ -0,0 +1,150 @@+import { eachDay, utcDayOf } from '#feedback/days'+import { createFeedbackDatabase } from '#feedback/db'+import type { FeedbackBindings, FeedbackServices } from '#feedback/env'+import { type CalendarDate, maxDate, minDate, parseDate } from '@internationalized/date'+import { compact } from 'es-toolkit'++import { cloudflareClient } from './cloudflare'+import { githubClient } from './github'+import { lastDay, replaceDays, type SnapshotStore } from './store'++/**+ * How far back the visit history is filled: Cloudflare's Free plan keeps 31 days and answers at+ * most 30 per query, so four weeks stays inside both.+ */+const BACKFILL_DAYS = 28+/** Days Cloudflare may still be counting; they are fetched again on the next runs. */+const SETTLING_DAYS = 3++export type SnapshotSource = 'github' | 'traffic' | 'site'++interface Run {+ store: SnapshotStore+ env: FeedbackBindings+ services: FeedbackServices+ today: CalendarDate+ /** The day that just ended, which this run records. */+ yesterday: CalendarDate+}++/** Stars, forks, and running download totals as of the end of yesterday. */+async function recordGithub({ store, env, services, yesterday }: Run) {+ const github = githubClient(services.fetch, env.GITHUB_REPOSITORY, env.GITHUB_TOKEN)+ const totals = await github.totals()+ const day = yesterday.toString()+ await replaceDays(+ store,+ 'githubStars',+ [yesterday],+ [{ day, stars: totals.stars, forks: totals.forks }]+ )+ await replaceDays(+ store,+ 'releaseDownloads',+ [yesterday],+ totals.downloads.map((file) => ({ day, ...file }))+ )+}++/** GitHub's last 14 days of views and clones, finished days only, and its referrer ranking. */+async function recordTraffic({ store, env, services, today, yesterday }: Run) {+ const github = githubClient(services.fetch, env.GITHUB_REPOSITORY, env.GITHUB_TOKEN)+ const traffic = await github.traffic()+ const finished = traffic.days.filter((day) => day.day.compare(today) < 0)+ const first = finished.at(0)+ // GitHub may not have counted yesterday yet; its window still ends there.+ const window = first ? eachDay(first.day, yesterday) : []+ await replaceDays(+ store,+ 'repoTraffic',+ window,+ finished.map((day) => ({ ...day, day: day.day.toString() }))+ )+ await replaceDays(+ store,+ 'repoReferrers',+ [yesterday],+ traffic.referrers.map((referrer) => ({ day: yesterday.toString(), ...referrer }))+ )+}++/**+ * Visits per hostname, and the website's pages and countries, for every day not recorded yet+ * plus the last few, which Cloudflare may still have been counting.+ */+async function recordSite({ store, env, services, yesterday }: Run) {+ const cloudflare = cloudflareClient(+ services.fetch,+ env.SITE_ZONE_ID,+ env.CLOUDFLARE_ANALYTICS_TOKEN+ )+ const hosts = env.SITE_HOSTS.split(',').map((host) => host.trim())+ const [website] = hosts+ const oldest = yesterday.subtract({ days: BACKFILL_DAYS - 1 })+ const recorded = await lastDay(store.db, 'siteVisits')+ const next = recorded ? parseDate(recorded).add({ days: 1 }) : oldest+ const settling = yesterday.subtract({ days: SETTLING_DAYS - 1 })+ // Start at the first missing day or the first still settling, whichever is earlier, but never+ // before the oldest day Cloudflare still has.+ const start = maxDate(minDate(next, settling), oldest) ?? oldest++ const days = eachDay(start, yesterday)+ const counted = await cloudflare.visits(start, yesterday, hosts)+ // Cloudflare leaves out a host's day without visits; it is recorded as zero, so every fetched+ // day has a row for every host and the newest one is truly where the records stop.+ const visits = days.flatMap((date) => {+ const day = date.toString()+ return hosts.map(+ (host) =>+ counted.find((row) => row.day === day && row.host === host) ?? { day, host, visits: 0 }+ )+ })+ if (website) {+ for (const day of days) {+ const { pages, countries } = await cloudflare.breakdown(day, website)+ await replaceDays(store, 'sitePages', [day], pages)+ await replaceDays(store, 'siteCountries', [day], countries)+ }+ }+ // Recorded last: the newest visit day is where the next run resumes, so a breakdown failing+ // partway leaves the whole range to be fetched again rather than skipped for good.+ await replaceDays(store, 'siteVisits', days, visits)+}++const SOURCES: [SnapshotSource, (run: Run) => Promise<void>][] = [+ ['github', recordGithub],+ ['traffic', recordTraffic],+ ['site', recordSite]+]++/**+ * The daily metrics snapshot, run just after midnight UTC. GitHub reports only running totals+ * and keeps traffic for 14 days, and Cloudflare keeps visits for 31, so these tables are the+ * only lasting history. Each source runs on its own: one failing still records the others, and+ * its gap is filled by the next run where the API still has the days.+ */+export async function runSnapshot(+ env: FeedbackBindings,+ services: FeedbackServices+): Promise<{ failed: SnapshotSource[] }> {+ const today = utcDayOf(services.now())+ const run: Run = {+ store: { db: createFeedbackDatabase(env.DB), d1: env.DB },+ env,+ services,+ today,+ yesterday: today.subtract({ days: 1 })+ }+ const failed = await Promise.all(+ SOURCES.map(async ([source, record]) => {+ try {+ await record(run)+ return undefined+ } catch (error) {+ console.error('Metrics snapshot failed', { source, error: String(error) })+ return source+ }+ })+ )+ return { failed: compact(failed) }+}diff --git a/packages/feedback/src/snapshot/store.ts b/packages/feedback/src/snapshot/store.tsnew file mode 100644index 000000000..5faa731b4--- /dev/null+++ b/packages/feedback/src/snapshot/store.ts@@ -0,0 +1,75 @@+import type { FeedbackDatabase } from '#feedback/db'+import type { D1Database } from '@cloudflare/workers-types/index'+import type { CalendarDate } from '@internationalized/date'+import { chunk } from 'es-toolkit'+import type { CompiledQuery, Insertable, Kysely } from 'kysely'++/** Kysely builds the statements; D1 runs a day's replacement as one batch, which is atomic. */+export interface SnapshotStore {+ db: Kysely<FeedbackDatabase>+ d1: D1Database+}++type SnapshotTable =+ | 'githubStars'+ | 'releaseDownloads'+ | 'repoTraffic'+ | 'repoReferrers'+ | 'siteVisits'+ | 'sitePages'+ | 'siteCountries'++// D1 binds at most 100 values per statement, so long lists go in several statements.+const MAX_BOUND_VALUES = 100++type ClearDays = (db: Kysely<FeedbackDatabase>, days: string[]) => CompiledQuery++// One query per table: Kysely cannot type a delete over a union of tables.+const CLEAR_DAYS: Record<SnapshotTable, ClearDays> = {+ githubStars: (db, days) => db.deleteFrom('githubStars').where('day', 'in', days).compile(),+ releaseDownloads: (db, days) =>+ db.deleteFrom('releaseDownloads').where('day', 'in', days).compile(),+ repoTraffic: (db, days) => db.deleteFrom('repoTraffic').where('day', 'in', days).compile(),+ repoReferrers: (db, days) => db.deleteFrom('repoReferrers').where('day', 'in', days).compile(),+ siteVisits: (db, days) => db.deleteFrom('siteVisits').where('day', 'in', days).compile(),+ sitePages: (db, days) => db.deleteFrom('sitePages').where('day', 'in', days).compile(),+ siteCountries: (db, days) => db.deleteFrom('siteCountries').where('day', 'in', days).compile()+}++/**+ * Makes `days` hold exactly `rows`: whatever was recorded for those days goes first, so an+ * entry the latest answer no longer has (a host without visits, a page out of the ranking, a+ * removed release file) does not linger, and a rerun corrects a day instead of adding to it.+ * The deletes and inserts run as one D1 batch, so a failure leaves the old rows in place.+ */+export async function replaceDays<T extends SnapshotTable>(+ { db, d1 }: SnapshotStore,+ table: T,+ days: CalendarDate[],+ rows: Insertable<FeedbackDatabase[T]>[]+): Promise<void> {+ const statements = chunk(+ days.map((day) => day.toString()),+ MAX_BOUND_VALUES+ ).map((batch) => CLEAR_DAYS[table](db, batch))+ const [first] = rows+ if (first) {+ const perStatement = Math.max(1, Math.floor(MAX_BOUND_VALUES / Object.keys(first).length))+ for (const batch of chunk(rows, perStatement))+ statements.push(db.insertInto(table).values(batch).compile())+ }+ if (!statements.length) return+ await d1.batch(statements.map((query) => d1.prepare(query.sql).bind(...query.parameters)))+}++/** The newest day recorded in a table, if any. */+export async function lastDay(+ db: Kysely<FeedbackDatabase>,+ table: 'siteVisits'+): Promise<string | undefined> {+ const row = await db+ .selectFrom(table)+ .select((eb) => eb.fn.max('day').as('day'))+ .executeTakeFirst()+ return row?.day ?? undefined+}diff --git a/packages/feedback/tests/harness.ts b/packages/feedback/tests/harness.tsindex cf99934f0..56b8106ba 100644--- a/packages/feedback/tests/harness.ts+++ b/packages/feedback/tests/harness.ts@@ -1,7 +1,8 @@import { readdir, readFile } from 'node:fs/promises'import { join } from 'node:path'-import { createFeedbackApp } from '#feedback/app'+import { createFeedbackApp, type FeedbackServices } from '#feedback/app'+import { maxBy } from 'es-toolkit'import { testClient } from 'hono/testing'import { getPlatformProxy } from 'wrangler'@@ -49,7 +50,7 @@ export async function createTestService() {await applyMigrations(proxy.env.DB)const fetchCalls: FetchCall[] = []- const responses = new Map<string, () => Response>()+ const responses = new Map<string, (call: FetchCall) => Response>()const sentEmails: EmailMessageBuilder[] = []let now = new Date('2026-10-12T09:00:00.000Z')const limiters = { ping: testLimiter(), ingest: testLimiter(), waitlist: testLimiter() }@@ -61,6 +62,8 @@ export async function createTestService() {TURNSTILE_SECRET: 'turnstile-secret',TURNSTILE_HOSTNAMES: 'openpencil.dev,app.openpencil.dev',EMAIL_FROM: '[email protected]',+ GITHUB_TOKEN: 'github-token',+ CLOUDFLARE_ANALYTICS_TOKEN: 'cloudflare-token',PING_LIMITER: limiters.ping,INGEST_LIMITER: limiters.ingest,WAITLIST_LIMITER: limiters.waitlist,@@ -73,15 +76,21 @@ export async function createTestService() {}}- const app = createFeedbackApp({+ const services: FeedbackServices = {async fetch(input, init) {const url = input instanceof Request ? input.url : String(input)- fetchCalls.push({ url, init })- const respond = [...responses].find(([prefix]) => url.startsWith(prefix))?.[1]- return respond ? respond() : new Response(null, { status: 200 })+ const call = { url, init }+ fetchCalls.push(call)+ // The most specific prefix answers, so a repository's own URL does not shadow its subpaths.+ const match = maxBy(+ [...responses].filter(([prefix]) => url.startsWith(prefix)),+ ([prefix]) => prefix.length+ )+ return match ? match[1](call) : new Response(null, { status: 200 })},now: () => now- })+ }+ const app = createFeedbackApp(services)const request = (path: string, init?: RequestInit) =>app.request(`https://feedback.openpencil.dev${path}`, init, env)@@ -90,12 +99,18 @@ export async function createTestService() {/** A client typed by the service's routes, as the app's is, sending into the app under test. */client: testClient(app, env),env,+ /** The outside world the app and the cron jobs see. */+ services,limiters,db: proxy.env.DB,fetchCalls,sentEmails,- /** Answers outside requests whose URL starts with `prefix`. */- respond(prefix: string, response: () => Response) {+ /** Forgets every answer registered with `respond`, so one test's answers do not reach the next. */+ clearResponses() {+ responses.clear()+ },+ /** Answers outside requests whose URL starts with `prefix`; the longest matching prefix wins. */+ respond(prefix: string, response: (call: FetchCall) => Response) {responses.set(prefix, response)},setNow(date: Date) {diff --git a/packages/feedback/tests/ping.test.ts b/packages/feedback/tests/ping.test.tsindex d7899b265..3a3df22ef 100644--- a/packages/feedback/tests/ping.test.ts+++ b/packages/feedback/tests/ping.test.ts@@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'import { createTestService, type TestService } from '#feedback-tests/harness'import type { Ping } from '#feedback/contract'+import { createFeedbackDatabase } from '#feedback/db'import { omit } from 'es-toolkit'const BUILD: Ping = {@@ -23,13 +24,12 @@ afterAll(async () => {await service.dispose()})-async function counts() {- const { results } = await service.db- .prepare(- 'SELECT day, version, platform, app, usage_stats AS usageStats, count FROM usage_pings ORDER BY version, usage_stats'- )- .all()- return results+function counts() {+ return createFeedbackDatabase(service.db)+ .selectFrom('usagePings')+ .select(['day', 'version', 'platform', 'app', 'usageStats', 'count'])+ .orderBy(['version', 'usageStats'])+ .execute()}describe('ping', () => {diff --git a/packages/feedback/tests/snapshot.test.ts b/packages/feedback/tests/snapshot.test.tsnew file mode 100644index 000000000..a850142ab--- /dev/null+++ b/packages/feedback/tests/snapshot.test.ts@@ -0,0 +1,354 @@+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'++import { type FetchCall, createTestService, type TestService } from '#feedback-tests/harness'+import { eachDay } from '#feedback/days'+import { createFeedbackDatabase } from '#feedback/db'+import { runSnapshot } from '#feedback/snapshot'+import { releaseFileKind } from '#feedback/snapshot/assets'+import { replaceDays } from '#feedback/snapshot/store'+import { parseDate } from '@internationalized/date'+import * as v from 'valibot'++let service: TestService++const REPO = 'https://api.github.com/repos/open-pencil/open-pencil'+const GRAPHQL = 'https://api.cloudflare.com/client/v4/graphql'+// The cron fires just after midnight, so the day that just ended is Oct 19.+const RUN_AT = new Date('2026-10-20T00:05:00.000Z')+const TABLES = [+ 'githubStars',+ 'releaseDownloads',+ 'repoTraffic',+ 'repoReferrers',+ 'siteVisits',+ 'sitePages',+ 'siteCountries'+] as const++const RELEASES = [+ {+ tag_name: 'v0.16.0',+ assets: [+ { name: 'OpenPencil_0.16.0_aarch64.dmg', download_count: 100 },+ { name: 'OpenPencil_aarch64.app.tar.gz', download_count: 300 },+ { name: 'OpenPencil_aarch64.app.tar.gz.sig', download_count: 6 },+ { name: 'OpenPencil_0.16.0_x64-setup.exe', download_count: 234 },+ { name: 'OpenPencil_0.16.0_x64_en-US.msi', download_count: 42 },+ { name: 'OpenPencil_0.16.0_amd64.deb', download_count: 72 },+ { name: 'latest.json', download_count: 1110 },+ { name: 'SHA256SUMS', download_count: 23 }+ ]+ },+ {+ tag_name: 'v0.15.1',+ assets: [{ name: 'OpenPencil_0.15.1_amd64.AppImage', download_count: 393 }]+ }+]++const VariablesSchema = v.object({ variables: v.record(v.string(), v.unknown()) })++/** The GraphQL variables a Cloudflare request carried. */+function variablesOf(call: FetchCall): Record<string, unknown> {+ const result = v.safeParse(+ v.pipe(v.string(), v.parseJson(), VariablesSchema),+ String(call.init?.body)+ )+ if (!result.success) throw new Error('Not a GraphQL request')+ return result.output.variables+}++const cloudflareCalls = () => service.fetchCalls.filter((call) => call.url === GRAPHQL)++function answerGithub() {+ service.respond(REPO, () => Response.json({ stargazers_count: 9120, forks_count: 933 }))+ service.respond(`${REPO}/releases`, () => Response.json(RELEASES))+ service.respond(`${REPO}/traffic/views`, () =>+ Response.json({+ views: [+ { timestamp: '2026-10-18T00:00:00Z', count: 900, uniques: 400 },+ { timestamp: '2026-10-19T00:00:00Z', count: 1200, uniques: 500 },+ { timestamp: '2026-10-20T00:00:00Z', count: 30, uniques: 20 }+ ]+ })+ )+ service.respond(`${REPO}/traffic/clones`, () =>+ Response.json({ clones: [{ timestamp: '2026-10-19T00:00:00Z', count: 80, uniques: 40 }] })+ )+ service.respond(`${REPO}/traffic/popular/referrers`, () =>+ Response.json([{ referrer: 'news.ycombinator.com', count: 700, uniques: 300 }])+ )+}++/** Answers every day with the same visits, and each breakdown with `pages` for the website. */+function answerCloudflare(pages: string[] = ['/', '/download']) {+ service.respond(GRAPHQL, (call) => answerOnce(call, pages))+}++function answerOnce(call: FetchCall, pages: string[] = ['/', '/download']): Response {+ const variables = variablesOf(call)+ if (typeof variables.start === 'string' && typeof variables.end === 'string') {+ const days = eachDay(parseDate(variables.start), parseDate(variables.end))+ const groups = days.map(String).flatMap((date) => [+ { sum: { visits: 4000 }, dimensions: { date, clientRequestHTTPHost: 'openpencil.dev' } },+ { sum: { visits: 1000 }, dimensions: { date, clientRequestHTTPHost: 'app.openpencil.dev' } }+ ])+ return Response.json({+ data: { viewer: { zones: [{ httpRequestsAdaptiveGroups: groups }] } }+ })+ }+ return Response.json({+ data: {+ viewer: {+ zones: [+ {+ pages: pages.map((clientRequestPath, index) => ({+ sum: { visits: 100 - index },+ dimensions: { clientRequestPath }+ })),+ countries: [+ { sum: { visits: 60 }, dimensions: { clientCountryName: 'US' } },+ { sum: { visits: 0 }, dimensions: { clientCountryName: 'XX' } }+ ]+ }+ ]+ }+ }+ })+}++/** The service's D1 through the same typed Kysely the Worker uses. */+const database = () => createFeedbackDatabase(service.db)++/** Every row of a snapshot table, in a fixed order. */+const rows = {+ githubStars: () => database().selectFrom('githubStars').selectAll().orderBy('day').execute(),+ releaseDownloads: () =>+ database().selectFrom('releaseDownloads').selectAll().orderBy(['tag', 'kind']).execute(),+ repoTraffic: () => database().selectFrom('repoTraffic').selectAll().orderBy('day').execute(),+ repoReferrers: () => database().selectFrom('repoReferrers').selectAll().orderBy('day').execute(),+ siteVisits: () =>+ database().selectFrom('siteVisits').selectAll().orderBy(['day', 'host']).execute(),+ siteCountries: () => database().selectFrom('siteCountries').selectAll().execute()+}++beforeAll(async () => {+ service = await createTestService()+})++beforeEach(async () => {+ for (const table of TABLES) await database().deleteFrom(table).execute()+ service.fetchCalls.length = 0+ service.clearResponses()+ service.setNow(RUN_AT)+ answerGithub()+ answerCloudflare()+})++afterAll(async () => {+ await service.dispose()+})++describe('release files', () => {+ test('counts installers, macOS updates, and update checks, and nothing else', () => {+ expect(releaseFileKind('OpenPencil_0.16.0_aarch64.dmg')).toBe('macos-installer')+ expect(releaseFileKind('OpenPencil_x64.app.tar.gz')).toBe('macos-update')+ expect(releaseFileKind('OpenPencil_0.16.0_arm64-setup.exe')).toBe('windows-installer')+ expect(releaseFileKind('OpenPencil-0.16.0-1.x86_64.rpm')).toBe('linux-installer')+ expect(releaseFileKind('latest.json')).toBe('update-check')+ expect(releaseFileKind('OpenPencil_x64.app.tar.gz.sig')).toBeUndefined()+ expect(releaseFileKind('SHA256SUMS')).toBeUndefined()+ })+})++describe('metrics snapshot', () => {+ test('records GitHub totals as of the end of the day that just ended', async () => {+ expect(await runSnapshot(service.env, service.services)).toEqual({ failed: [] })+ expect(await rows.githubStars()).toEqual([{ day: '2026-10-19', stars: 9120, forks: 933 }])+ expect(await rows.releaseDownloads()).toEqual([+ { day: '2026-10-19', tag: 'v0.15.1', kind: 'linux-installer', downloads: 393 },+ { day: '2026-10-19', tag: 'v0.16.0', kind: 'linux-installer', downloads: 72 },+ { day: '2026-10-19', tag: 'v0.16.0', kind: 'macos-installer', downloads: 100 },+ { day: '2026-10-19', tag: 'v0.16.0', kind: 'macos-update', downloads: 300 },+ { day: '2026-10-19', tag: 'v0.16.0', kind: 'update-check', downloads: 1110 },+ { day: '2026-10-19', tag: 'v0.16.0', kind: 'windows-installer', downloads: 276 }+ ])+ const github = service.fetchCalls.find((call) => call.url === REPO)+ expect(new Headers(github?.init?.headers).get('Authorization')).toBe('Bearer github-token')+ })++ test('reads every page of releases', async () => {+ const page = (tag: string) => ({+ tag_name: tag,+ assets: [{ name: 'latest.json', download_count: 1 }]+ })+ service.respond(`${REPO}/releases?per_page=100&page=1`, () =>+ Response.json(Array.from({ length: 100 }, (_, index) => page(`v0.${index}.0`)))+ )+ service.respond(`${REPO}/releases?per_page=100&page=2`, () => Response.json([page('v1.0.0')]))+ await runSnapshot(service.env, service.services)+ expect(await rows.releaseDownloads()).toHaveLength(101)+ })++ test('keeps finished traffic days only and replaces the day’s referrers', async () => {+ await database()+ .insertInto('repoReferrers')+ .values({ day: '2026-10-19', referrer: 'github.com', views: 1, uniqueViews: 1 })+ .execute()+ await runSnapshot(service.env, service.services)+ expect(await rows.repoTraffic()).toEqual([+ { day: '2026-10-18', views: 900, uniqueViews: 400, clones: 0, uniqueClones: 0 },+ { day: '2026-10-19', views: 1200, uniqueViews: 500, clones: 80, uniqueClones: 40 }+ ])+ expect(await rows.repoReferrers()).toEqual([+ { day: '2026-10-19', referrer: 'news.ycombinator.com', views: 700, uniqueViews: 300 }+ ])+ })++ test('fills four weeks of visits on the first run', async () => {+ await runSnapshot(service.env, service.services)+ const [visits, ...breakdowns] = cloudflareCalls()+ expect(visits && variablesOf(visits)).toMatchObject({+ zone: '6a1c3366d17c9cdefab66cf5dd4e1cbe',+ start: '2026-09-22',+ end: '2026-10-19',+ hosts: ['openpencil.dev', 'app.openpencil.dev']+ })+ expect(new Headers(visits?.init?.headers).get('Authorization')).toBe('Bearer cloudflare-token')+ expect(breakdowns.map((call) => variablesOf(call).host)).toEqual(+ Array.from({ length: 28 }, () => 'openpencil.dev')+ )+ expect(await rows.siteVisits()).toHaveLength(56)+ expect(await rows.siteCountries()).toHaveLength(28)+ })++ test('the next night fetches only the days Cloudflare may still be counting', async () => {+ await runSnapshot(service.env, service.services)+ service.fetchCalls.length = 0+ service.setNow(new Date(Date.parse(RUN_AT.toISOString()) + 24 * 60 * 60 * 1000))+ await runSnapshot(service.env, service.services)+ const [visits] = cloudflareCalls()+ expect(visits && variablesOf(visits)).toMatchObject({ start: '2026-10-18', end: '2026-10-20' })+ expect(cloudflareCalls()).toHaveLength(4)+ })++ test('a gap longer than the settling days is filled from where the records stop', async () => {+ const recorded = parseDate('2026-10-19').subtract({ days: 10 })+ await database()+ .insertInto('siteVisits')+ .values({ day: recorded.toString(), host: 'openpencil.dev', visits: 1 })+ .execute()+ await runSnapshot(service.env, service.services)+ const [visits] = cloudflareCalls()+ expect(visits && variablesOf(visits)).toMatchObject({+ start: recorded.add({ days: 1 }).toString()+ })+ })++ test('a page that drops out of the ranking is removed when its day is fetched again', async () => {+ await runSnapshot(service.env, service.services)+ answerCloudflare(['/'])+ await runSnapshot(service.env, service.services)+ const pages = await database()+ .selectFrom('sitePages')+ .select('path')+ .where('day', '=', '2026-10-19')+ .execute()+ expect(pages).toEqual([{ path: '/' }])+ })++ test('a breakdown failing partway records no visits, so the next run fetches the range again', async () => {+ service.respond(GRAPHQL, (call) => {+ if (variablesOf(call).day === '2026-09-26') return new Response('busy', { status: 503 })+ return answerOnce(call)+ })+ expect(await runSnapshot(service.env, service.services)).toEqual({ failed: ['site'] })+ expect(await rows.siteVisits()).toEqual([])+ answerCloudflare()+ service.fetchCalls.length = 0+ await runSnapshot(service.env, service.services)+ const [visits] = cloudflareCalls()+ expect(visits && variablesOf(visits)).toMatchObject({ start: '2026-09-22' })+ expect(await rows.siteCountries()).toHaveLength(28)+ })++ test('a day Cloudflare leaves out is recorded as zero visits', async () => {+ service.respond(GRAPHQL, (call) => {+ const variables = variablesOf(call)+ if (typeof variables.start !== 'string') return answerOnce(call)+ // Only the website had visits, and only on the last day.+ const group = {+ sum: { visits: 3000 },+ dimensions: { date: '2026-10-19', clientRequestHTTPHost: 'openpencil.dev' }+ }+ return Response.json({+ data: { viewer: { zones: [{ httpRequestsAdaptiveGroups: [group] }] } }+ })+ })+ await runSnapshot(service.env, service.services)+ const visits = await rows.siteVisits()+ expect(visits).toHaveLength(56)+ expect(visits.filter((row) => row.visits > 0)).toEqual([+ { day: '2026-10-19', host: 'openpencil.dev', visits: 3000 }+ ])+ })++ test('a host missing from a rerun is set back to zero for that day', async () => {+ await database()+ .insertInto('siteVisits')+ .values([+ { day: '2026-10-19', host: 'app.openpencil.dev', visits: 999 },+ { day: '2026-10-18', host: 'openpencil.dev', visits: 5 }+ ])+ .execute()+ service.respond(GRAPHQL, (call) => {+ const variables = variablesOf(call)+ if (typeof variables.start !== 'string') return answerOnce(call)+ const group = {+ sum: { visits: 4000 },+ dimensions: { date: '2026-10-19', clientRequestHTTPHost: 'openpencil.dev' }+ }+ return Response.json({+ data: { viewer: { zones: [{ httpRequestsAdaptiveGroups: [group] }] } }+ })+ })+ await runSnapshot(service.env, service.services)+ const visits = await rows.siteVisits()+ expect(visits.filter((row) => row.day >= '2026-10-18')).toEqual([+ { day: '2026-10-18', host: 'app.openpencil.dev', visits: 0 },+ { day: '2026-10-18', host: 'openpencil.dev', visits: 0 },+ { day: '2026-10-19', host: 'app.openpencil.dev', visits: 0 },+ { day: '2026-10-19', host: 'openpencil.dev', visits: 4000 }+ ])+ })++ test('a replacement that fails to insert leaves the day as it was', async () => {+ await database()+ .insertInto('siteVisits')+ .values({ day: '2026-10-19', host: 'openpencil.dev', visits: 4258 })+ .execute()+ const store = { db: database(), d1: service.db }+ // Two rows with the same key: the insert fails after the delete was already queued.+ const duplicate = { day: '2026-10-19', host: 'openpencil.dev', visits: 1 }+ await expect(+ replaceDays(store, 'siteVisits', [parseDate('2026-10-19')], [duplicate, duplicate])+ ).rejects.toThrow()+ expect(await rows.siteVisits()).toEqual([+ { day: '2026-10-19', host: 'openpencil.dev', visits: 4258 }+ ])+ })++ test('one source failing still records the others', async () => {+ service.respond(REPO, () => new Response('unavailable', { status: 503 }))+ service.respond(`${REPO}/releases`, () => new Response('unavailable', { status: 503 }))+ expect(await runSnapshot(service.env, service.services)).toEqual({ failed: ['github'] })+ expect(await rows.githubStars()).toEqual([])+ expect(await rows.repoTraffic()).toHaveLength(2)+ expect(await rows.siteVisits()).toHaveLength(56)+ })++ test('an answer of the wrong shape fails that source rather than recording it', async () => {+ service.respond(GRAPHQL, () => Response.json({ data: null, errors: [{ message: 'quota' }] }))+ expect(await runSnapshot(service.env, service.services)).toEqual({ failed: ['site'] })+ expect(await rows.siteVisits()).toEqual([])+ })+})diff --git a/packages/feedback/tests/waitlist.test.ts b/packages/feedback/tests/waitlist.test.tsindex e821319a5..443db620b 100644--- a/packages/feedback/tests/waitlist.test.ts+++ b/packages/feedback/tests/waitlist.test.ts@@ -2,6 +2,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:tesimport { createTestService, type TestService } from '#feedback-tests/harness'import type { WaitlistSignup } from '#feedback/contract'+import { createFeedbackDatabase } from '#feedback/db'import { runRetention } from '#feedback/retention'let service: TestService@@ -17,11 +18,12 @@ function turnstile(result: { success: boolean; hostname?: string; action?: strinservice.respond('https://challenges.cloudflare.com/', () => Response.json(result))}-async function rows() {- const { results } = await service.db- .prepare('SELECT email, note, confirmed_at AS confirmedAt FROM waitlist ORDER BY id')- .all()- return results+function rows() {+ return createFeedbackDatabase(service.db)+ .selectFrom('waitlist')+ .select(['email', 'note', 'confirmedAt'])+ .orderBy('id')+ .execute()}function linkFrom(text: string | undefined, path: string): URL {@@ -40,7 +42,7 @@ beforeAll(async () => {})beforeEach(async () => {- await service.db.prepare('DELETE FROM waitlist').run()+ await createFeedbackDatabase(service.db).deleteFrom('waitlist').execute()service.sentEmails.length = 0service.setNow(START)turnstile({ success: true, hostname: 'openpencil.dev', action: 'waitlist' })diff --git a/packages/feedback/worker-configuration.d.ts b/packages/feedback/worker-configuration.d.tsindex 1f3703057..8b3dd086a 100644--- a/packages/feedback/worker-configuration.d.ts+++ b/packages/feedback/worker-configuration.d.ts@@ -1,5 +1,5 @@/* eslint-disable */-// Generated by Wrangler by running `wrangler types --include-runtime=false` (hash: d8d43b461a15bf47e1b2c7555a4a6760)+// Generated by Wrangler by running `wrangler types --include-runtime=false` (hash: 160eb360e2ef5a7cbd4000dba7fe6462)interface __BaseEnv_Env {DB: D1Database;EMAIL: SendEmail;@@ -9,8 +9,13 @@ interface __BaseEnv_Env {POSTHOG_HOST: "https://eu.i.posthog.com";EMAIL_FROM: "[email protected]";TURNSTILE_HOSTNAMES: "openpencil.dev,app.openpencil.dev";+ GITHUB_REPOSITORY: "open-pencil/open-pencil";+ SITE_ZONE_ID: "6a1c3366d17c9cdefab66cf5dd4e1cbe";+ SITE_HOSTS: "openpencil.dev,app.openpencil.dev";POSTHOG_KEY: string;TURNSTILE_SECRET: string;+ GITHUB_TOKEN: string;+ CLOUDFLARE_ANALYTICS_TOKEN: string;}declare namespace Cloudflare {interface GlobalProps {@@ -23,7 +28,7 @@ type StringifyValues<EnvType extends Record<string, unknown>> = {[Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string;};declare namespace NodeJS {- interface ProcessEnv extends StringifyValues<Pick<Cloudflare.Env, "POSTHOG_HOST" | "EMAIL_FROM" | "TURNSTILE_HOSTNAMES" | "POSTHOG_KEY" | "TURNSTILE_SECRET">> {}+ interface ProcessEnv extends StringifyValues<Pick<Cloudflare.Env, "POSTHOG_HOST" | "EMAIL_FROM" | "TURNSTILE_HOSTNAMES" | "GITHUB_REPOSITORY" | "SITE_ZONE_ID" | "SITE_HOSTS" | "POSTHOG_KEY" | "TURNSTILE_SECRET" | "GITHUB_TOKEN" | "CLOUDFLARE_ANALYTICS_TOKEN">> {}}declare module "*.txt" {const value: string;diff --git a/packages/feedback/wrangler.jsonc b/packages/feedback/wrangler.jsoncindex 296db4bf4..6019f4b75 100644--- a/packages/feedback/wrangler.jsonc+++ b/packages/feedback/wrangler.jsonc@@ -10,7 +10,12 @@"POSTHOG_HOST": "https://eu.i.posthog.com","EMAIL_FROM": "[email protected]",// The website's form and the web app's Settings; the desktop app opens the website.- "TURNSTILE_HOSTNAMES": "openpencil.dev,app.openpencil.dev"+ "TURNSTILE_HOSTNAMES": "openpencil.dev,app.openpencil.dev",+ // What the daily metrics snapshot records: this repository's stars, downloads, and traffic,+ // and Cloudflare's visits to these hostnames in the openpencil.dev zone.+ "GITHUB_REPOSITORY": "open-pencil/open-pencil",+ "SITE_ZONE_ID": "6a1c3366d17c9cdefab66cf5dd4e1cbe",+ "SITE_HOSTS": "openpencil.dev,app.openpencil.dev"},// Created with `wrangler d1 create feedback --jurisdiction eu`, so rows stay in the EU."d1_databases": [@@ -28,7 +33,10 @@{ "name": "WAITLIST_LIMITER", "namespace_id": "1003", "simple": { "limit": 5, "period": 60 } }],"send_email": [{ "name": "EMAIL" }],- "triggers": { "crons": ["17 3 * * *"] },+ // Just after midnight UTC the metrics snapshot records the day that ended; later, retention runs.+ "triggers": { "crons": ["5 0 * * *", "17 3 * * *"] },// Set with `wrangler secret put`; locally from `.dev.vars` (see `.dev.vars.example`).- "secrets": { "required": ["POSTHOG_KEY", "TURNSTILE_SECRET"] }+ "secrets": {+ "required": ["POSTHOG_KEY", "TURNSTILE_SECRET", "GITHUB_TOKEN", "CLOUDFLARE_ANALYTICS_TOKEN"]+ }}diff --git a/tools/checks/lint/src/plugin.ts b/tools/checks/lint/src/plugin.tsindex 70306073f..ae4ddfa77 100644--- a/tools/checks/lint/src/plugin.ts+++ b/tools/checks/lint/src/plugin.ts@@ -1,6 +1,5 @@import {noCloudDynamicHTMLRule,- noCloudRawSQLRule,noCloudSecretInQueryRule,noCloudSecretLoggingRule} from '#lint/rules/cloud.ts'@@ -16,6 +15,7 @@ import { noModuleMockingRule } from '#lint/rules/quality/module-mocking.ts'import { noReduceAccumulatorCopyRule } from '#lint/rules/quality/reduce-accumulator-copy.ts'import { noUnknownTailwindClassesRule } from '#lint/rules/quality/tailwind-classes.ts'import { noWidenThenAssertRule } from '#lint/rules/quality/widen-then-assert.ts'+import { noRawSQLRule } from '#lint/rules/sql.ts'import { normalizedFilename } from '#lint/support/context.ts'import type { PluginRule, RuleDefinition } from '#lint/support/types.ts'import type { TSESTree } from '@typescript-eslint/utils'@@ -506,7 +506,7 @@ const plugin = {'no-top-level-prefixed-test-files': noTopLevelPrefixedTestFiles,'no-conditional-object-spreads': noConditionalObjectSpreads,'no-hand-rolled-base64': noHandRolledBase64Rule,- 'no-cloud-raw-sql': noCloudRawSQLRule,+ 'no-raw-sql': noRawSQLRule,'no-cloud-dynamic-html': noCloudDynamicHTMLRule,'no-cloud-secret-in-query': noCloudSecretInQueryRule,'no-cloud-secret-logging': noCloudSecretLoggingRule,diff --git a/tools/checks/lint/src/rules/cloud.ts b/tools/checks/lint/src/rules/cloud.tsindex 89b1e4164..e16000079 100644--- a/tools/checks/lint/src/rules/cloud.ts+++ b/tools/checks/lint/src/rules/cloud.ts@@ -64,30 +64,6 @@ function loggedNames(node: ESTree.Node | null): string[] {}}-/** Kysely's typed builder everywhere; raw `sql` templates only in migrations and reviewed helpers. */-export const noCloudRawSQLRule = defineRule({- meta: {- type: 'problem',- docs: { description: 'Disallow raw sql templates in the Cloud server.' },- messages: {- raw: "Use Kysely's typed query builder; raw SQL belongs in migrations and the reviewed helpers in db/expressions.ts."- }- },- createOnce(context) {- return {- TaggedTemplateExpression(node) {- const { tag } = node- const named =- (tag.type === 'Identifier' && tag.name === 'sql') ||- (tag.type === 'MemberExpression' &&- tag.object.type === 'Identifier' &&- tag.object.name === 'sql')- if (named) context.report({ node, messageId: 'raw' })- }- }- }-})-/** Cloud APIs answer with typed JSON or a constant body, never HTML assembled from values. */export const noCloudDynamicHTMLRule = defineRule({meta: {diff --git a/tools/checks/lint/src/rules/sql.ts b/tools/checks/lint/src/rules/sql.tsnew file mode 100644index 000000000..01587b582--- /dev/null+++ b/tools/checks/lint/src/rules/sql.ts@@ -0,0 +1,28 @@+import { defineRule } from '@oxlint/plugins'++/**+ * Kysely's typed builder everywhere; raw `sql` templates only in migrations and reviewed helpers.+ * Applied to the Cloud server and the feedback Worker in oxlint.json.+ */+export const noRawSQLRule = defineRule({+ meta: {+ type: 'problem',+ docs: { description: 'Disallow raw sql templates in server code that uses Kysely.' },+ messages: {+ raw: "Use Kysely's typed query builder; raw SQL belongs in migrations and reviewed helpers."+ }+ },+ createOnce(context) {+ return {+ TaggedTemplateExpression(node) {+ const { tag } = node+ const named =+ (tag.type === 'Identifier' && tag.name === 'sql') ||+ (tag.type === 'MemberExpression' &&+ tag.object.type === 'Identifier' &&+ tag.object.name === 'sql')+ if (named) context.report({ node, messageId: 'raw' })+ }+ }+ }+})diff --git a/tools/checks/lint/tests/cloud.test.ts b/tools/checks/lint/tests/cloud.test.tsindex cf41a1802..b438174a6 100644--- a/tools/checks/lint/tests/cloud.test.ts+++ b/tools/checks/lint/tests/cloud.test.ts@@ -6,22 +6,6 @@ async function diagnostics(rule: string, source: string) {return ruleDiagnostics(await lint(source, { [`open-pencil/${rule}`]: 'error' }), rule)}-describe('no-cloud-raw-sql', () => {- test.each(['sql`select 1`', 'sql.raw`drop table x`', 'sql<number>`count(*)`'])(- 'rejects %s',- async (source) => {- expect(await diagnostics('no-cloud-raw-sql', source)).toHaveLength(1)- }- )-- test.each(["db.selectFrom('documents').selectAll()", 'html`<p></p>`'])(- 'accepts %s',- async (source) => {- expect(await diagnostics('no-cloud-raw-sql', source)).toHaveLength(0)- }- )-})-describe('no-cloud-dynamic-html', () => {test.each(['c.html(page)', 'c.html(`<p>` + name + `</p>`)', "c.html('<p>' + name + '</p>')"])('rejects %s',diff --git a/tools/checks/lint/tests/sql.test.ts b/tools/checks/lint/tests/sql.test.tsnew file mode 100644index 000000000..22c4d6a6f--- /dev/null+++ b/tools/checks/lint/tests/sql.test.ts@@ -0,0 +1,23 @@+import { describe, expect, test } from 'bun:test'++import { lint, ruleDiagnostics } from './helpers/lint.ts'++async function diagnostics(source: string) {+ return ruleDiagnostics(await lint(source, { 'open-pencil/no-raw-sql': 'error' }), 'no-raw-sql')+}++describe('no-raw-sql', () => {+ test.each(['sql`select 1`', 'sql.raw`drop table x`', 'sql<number>`count(*)`'])(+ 'rejects %s',+ async (source) => {+ expect(await diagnostics(source)).toHaveLength(1)+ }+ )++ test.each(["db.selectFrom('documents').selectAll()", 'html`<p></p>`'])(+ 'accepts %s',+ async (source) => {+ expect(await diagnostics(source)).toHaveLength(0)+ }+ )+})