Skip to content

The Open Standard for Generative UI

4.5 KB
Raw Blame
// Regenerates the static tweet-wall data (app/(home)/data/home-tweets-static.ts)
// AND downloads the avatars / media into public/tweets/, so the wall has zero
// runtime dependency on Twitter. Data comes from Twitter's public syndication
// endpoint (cdn.syndication.twimg.com, no API key).
//
// Usage:
//   pnpm --filter @openuidev/docs fetch:tweets     (or: cd docs && pnpm fetch:tweets)
//
// To change WHICH tweets appear, edit the ID list in
//   app/(home)/data/home-tweet-embeds.ts
// (the single source shared with the marquee) and re-run this script.

import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const EMBEDS_FILE = resolve(here, "../app/(home)/data/home-tweet-embeds.ts");
const OUT_FILE = resolve(here, "../app/(home)/data/home-tweets-static.ts");
const PUBLIC_DIR = resolve(here, "../public/tweets");

mkdirSync(PUBLIC_DIR, { recursive: true });

// Read the canonical tweet ID list (entries look like `{ id: "123...", ... }`).
const embedsSrc = readFileSync(EMBEDS_FILE, "utf8");
const IDS = [...embedsSrc.matchAll(/id:\s*"(\d+)"/g)].map((m) => m[1]);
if (IDS.length === 0) {
  console.error(`No tweet IDs found in ${EMBEDS_FILE}`);
  process.exit(1);
}

const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, "");
const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15";
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
const fmtDate = (iso) => {
  const d = new Date(iso);
  return `${MONTHS[d.getUTCMonth()]} ${d.getUTCDate()}, ${d.getUTCFullYear()}`;
};
const extOf = (url) => (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "jpg").toLowerCase();

async function download(url, filename) {
  const res = await fetch(url, { headers: { "User-Agent": UA } });
  if (!res.ok) throw new Error(`${url} -> HTTP ${res.status}`);
  writeFileSync(resolve(PUBLIC_DIR, filename), Buffer.from(await res.arrayBuffer()));
  return `/tweets/${filename}`;
}

const out = [];
for (const id of IDS) {
  const r = await fetch(
    `https://cdn.syndication.twimg.com/tweet-result?id=${id}&token=${token(id)}&lang=en`,
    { headers: { "User-Agent": UA } },
  );
  if (!r.ok) {
    console.error(`  ${id}: HTTP ${r.status} (skipped)`);
    continue;
  }
  const d = await r.json();
  const md = d.mediaDetails || [];
  const handle = d.user?.screen_name || id;

  let text = (d.text || "").trim();
  for (const u of d.entities?.urls || []) {
    if (u.url && u.display_url) text = text.split(u.url).join(u.display_url);
  }
  if (md.length) text = text.replace(/\s*https:\/\/t\.co\/\S+\s*$/, "").trim();

  const avatarUrl = (d.user?.profile_image_url_https || "").replace("_normal.", "_bigger.");

  const entry = {
    id,
    name: d.user?.name || "",
    handle,
    verified: !!(d.user?.is_blue_verified || d.user?.verified),
    avatar: avatarUrl ? await download(avatarUrl, `avatar-${handle}.${extOf(avatarUrl)}`) : "",
    text,
    date: d.created_at ? fmtDate(d.created_at) : "",
    likes: d.favorite_count ?? 0,
    replies: d.conversation_count ?? 0,
  };
  // Omit `media` entirely when absent so the generated literal stays assignable to
  // the optional `media?` field (a `media: null` would fail tsc).
  if (md.length) {
    const m = md[0];
    entry.media = {
      type: m.type === "photo" ? "photo" : "video",
      url: await download(m.media_url_https, `media-${id}.${extOf(m.media_url_https)}`),
    };
  }
  out.push(entry);
}

const ts = `// Static snapshot of the home tweet wall, generated by scripts/fetch-static-tweets.mjs.
// Avatars/media live in public/tweets/ (downloaded by the script, no Twitter dependency).
// To refresh, or to change which tweets appear (edit the IDs in home-tweet-embeds.ts),
// run \`pnpm fetch:tweets\`. Generated file, do not edit by hand.
export type StaticTweet = {
  id: string;
  name: string;
  handle: string;
  verified: boolean;
  avatar: string;
  text: string;
  date: string;
  likes: number;
  replies: number;
  media?: { type: "photo" | "video"; url: string };
};

export const STATIC_TWEETS: StaticTweet[] = ${JSON.stringify(out, null, 2)};
`;

writeFileSync(OUT_FILE, ts);
console.log(`Wrote ${out.length} tweets to ${OUT_FILE}`);
out.forEach((t) =>
  console.log(`  @${t.handle.padEnd(15)} likes=${String(t.likes).padStart(3)} replies=${String(t.replies).padStart(2)} media=${t.media?.type || "-"} avatar=${t.avatar}`),
);