OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M7-008): misc + tools modules into core + M7 wave checkpoint
ralph0eabc52f49e5
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 52ee3bfa0..46becd36f 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -38,7 +38,6 @@"@openpanel/trpc": "workspace:*","@openpanel/validation": "workspace:*","@trpc/server": "^11.17.0",- "cheerio": "^1.0.0","fast-json-stable-hash": "^1.0.3","fastify": "^5.8.5","fastify-metrics": "^12.1.0",@@ -50,7 +49,6 @@"ramda": "^0.29.1","sharp": "^0.33.5","source-map-support": "^0.5.21",- "url-metadata": "^5.4.1","uuid": "^9.0.1","zod": "catalog:"},diff --git a/apps/api/src/controllers/misc.controller.ts b/apps/api/src/controllers/misc.controller.tsindex be1c231a1..d73e655c7 100644--- a/apps/api/src/controllers/misc.controller.ts+++ b/apps/api/src/controllers/misc.controller.ts@@ -1,360 +1,54 @@-import crypto from 'node:crypto';+// Dissolved into @openpanel/core's misc module (M7-008): favicon/OG proxy,+// stats and geo lookup moved to+// packages/core/src/modules/misc/misc.service.ts. This file stays+// (DELEGATE PATTERN) — it is `misc.router.ts`'s Fastify glue, a thin wrapper+// around the core functions, same shape as `manage.controller.ts` (M6-002).+//+// `GET /misc/og/clear` and `/misc/favicon/clear` are dropped, not ported+// (ADR-015 entry #6: RULED + DEAD — `docs/ANSWERS.md` §1.4 confirms nothing+// depends on them).import {- DEFAULT_IP_HEADER_ORDER,- type GeoLocation,- getClientIpFromHeaders,- getGeoLocation,+ getFavicon as getFaviconCore,+ getGeoReport,+ getOgImage as getOgImageCore,+ getStats,+ type ImageAssetResult,+ insertPingRecord,} from '@openpanel/core';-import { ch, chQuery, formatClickhouseDate, TABLE_NAMES } from '@openpanel/db';-import { getCache, getRedisCache } from '@openpanel/redis';import type { FastifyReply, FastifyRequest } from 'fastify';-import {- ALLOWED_IMAGE_CONTENT_TYPES,- normalizeContentType,- processImage,- processOgImage,-} from '@/utils/image-proxy';-import { logger } from '@/utils/logger';-import { parseUrlMeta } from '@/utils/parseUrlMeta';-import {- assertPublicUrl,- BlockedUrlError,- safeFetch,-} from '@/utils/safe-fetch';interface GetFaviconParams {url: string;}-// Configuration-const TTL_SECONDS = 60 * 60 * 24; // 24h-const MAX_BYTES = 1_000_000; // 1MB cap-const USER_AGENT = 'OpenPanel-FaviconProxy/1.0 (+https://openpanel.dev)';--// Helper functions-function createCacheKey(url: string, prefix = 'favicon'): string {- const hash = crypto.createHash('sha256').update(url).digest('hex');- // v3: entries written before the SVG/raw-passthrough fix could hold- // attacker-controlled bytes with an attacker-chosen content type, so the- // old namespace is abandoned rather than served from.- return `${prefix}:v3:${hash}`;-}--/**- * Shape check only. The destination is validated by `assertPublicUrl` /- * `safeFetch` right before each outbound request, because a hostname can- * resolve differently between validation and connection.- */-function validateUrl(raw?: string): URL | null {- try {- if (!raw) {- throw new Error('Missing ?url');- }- const url = new URL(raw);- if (url.protocol !== 'http:' && url.protocol !== 'https:') {- throw new Error('Only http/https URLs are allowed');- }- return url;- } catch (error) {- return null;- }-}--// Binary cache functions (more efficient than base64)-async function getFromCacheBinary(- key: string-): Promise<{ buffer: Buffer; contentType: string } | null> {- const redis = getRedisCache();- const [bufferBase64, contentType] = await Promise.all([- redis.get(key),- redis.get(`${key}:ctype`),- ]);-- if (!(bufferBase64 && contentType)) {- return null;+function sendImageAsset(reply: FastifyReply, result: ImageAssetResult) {+ for (const [name, value] of Object.entries(result.headers)) {+ reply.header(name, value);}- return { buffer: Buffer.from(bufferBase64, 'base64'), contentType };-}--async function setToCacheBinary(- key: string,- buffer: Buffer,- contentType: string-): Promise<void> {- const redis = getRedisCache();- await Promise.all([- redis.set(key, buffer.toString('base64'), 'EX', TTL_SECONDS),- redis.set(`${key}:ctype`, contentType, 'EX', TTL_SECONDS),- ]);-}--// Fetch image with SSRF protection, timeout and size limits-async function fetchImage(- url: URL-): Promise<{ buffer: Buffer; contentType: string; status: number }> {- try {- // `safeFetch` validates and pins every hop, so a redirect cannot be used- // to reach an internal address after the initial URL checks out.- const result = await safeFetch(url, {- timeoutMs: 10_000,- maxBytes: MAX_BYTES,- headers: {- 'user-agent': USER_AGENT,- accept: 'image/*,*/*;q=0.8',- },- });-- if (result.status !== 200) {- return {- buffer: Buffer.alloc(0),- contentType: 'text/plain',- status: result.status,- };- }-- const contentType = normalizeContentType(- result.headers.get('content-type')- );- if (!ALLOWED_IMAGE_CONTENT_TYPES.has(contentType)) {- logger.debug(- { url: url.toString(), contentType },- 'Refusing non-image response'- );- return {- buffer: Buffer.alloc(0),- contentType: 'text/plain',- status: 415,- };- }-- return { buffer: result.body, contentType, status: 200 };- } catch (error) {- if (error instanceof BlockedUrlError) {- logger.warn(- { url: url.toString(), reason: error.message },- 'Blocked image fetch'- );- }- return { buffer: Buffer.alloc(0), contentType: 'text/plain', status: 500 };+ if (result.status === 200) {+ return reply.status(200).send(result.buffer);}-}--/**- * These bytes come from a third-party server, so make sure a browser treats- * them strictly as an image: no MIME sniffing, no scripts, no embedding- * privileges on the API origin.- */-function setImageSecurityHeaders(reply: FastifyReply, contentType: string) {- reply.header('Content-Type', contentType);- reply.header('X-Content-Type-Options', 'nosniff');- reply.header('Content-Security-Policy', "default-src 'none'; sandbox");- reply.header('Cross-Origin-Resource-Policy', 'cross-origin');-}--// Check if URL is a direct image-function isDirectImage(url: URL): boolean {- const imageExtensions = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];- return (- imageExtensions.some((ext) => url.pathname.endsWith(`.${ext}`)) ||- url.toString().includes('googleusercontent.com')- );+ return reply.status(result.status).send(result.body);}export async function getFavicon(- request: FastifyRequest<{- Querystring: GetFaviconParams;- }>,+ request: FastifyRequest<{ Querystring: GetFaviconParams }>,reply: FastifyReply) {- try {- logger.info({ url: request.query.url }, 'getFavicon');- const url = validateUrl(request.query.url);- if (!url) {- return reply- .status(404)- .header('Content-Type', 'text/plain')- .send('Not found');- }-- // Check the destination before anything else, so a blocked host is not- // leaked to the DuckDuckGo fallback below either.- await assertPublicUrl(url);-- const cacheKey = createCacheKey(url.toString());-- // Check cache first- const cached = await getFromCacheBinary(cacheKey);- if (cached) {- setImageSecurityHeaders(reply, cached.contentType);- reply.header('Cache-Control', 'public, max-age=604800, immutable');- return reply.send(cached.buffer);- }-- let imageUrl: URL;- // If it's a direct image URL, use it directly- if (isDirectImage(url)) {- imageUrl = url;- } else {- logger.info({ url: url.toString() }, 'before parseUrlMeta');- // For website URLs, extract favicon from HTML- const meta = await parseUrlMeta(url.toString());- logger.info(- { url: url.toString(), favicon: meta?.favicon },- 'parseUrlMeta result'- );- if (meta?.favicon) {- imageUrl = new URL(meta.favicon);- } else {- // Try standard favicon location first- const { origin } = url;- imageUrl = new URL(`${origin}/favicon.ico`);- }- }-- logger.info(- {- originalUrl: url.toString(),- imageUrl: imageUrl.toString(),- },- 'Fetching favicon'- );-- // Fetch the image- let { buffer, contentType, status } = await fetchImage(imageUrl);-- logger.info(- {- originalUrl: url.toString(),- imageUrl: imageUrl.toString(),- status,- bufferLength: buffer.length,- contentType,- },- 'Favicon fetch result'- );-- // If the direct favicon fetch failed and it's not from DuckDuckGo's service,- // try DuckDuckGo's favicon service as a fallback- if (buffer.length === 0 && !imageUrl.hostname.includes('duckduckgo.com')) {- const { hostname } = url;- const duckduckgoUrl = new URL(- `https://icons.duckduckgo.com/ip3/${hostname}.ico`- );-- logger.info(- {- originalUrl: url.toString(),- duckduckgoUrl: duckduckgoUrl.toString(),- },- 'Trying DuckDuckGo favicon service'- );-- const duckduckgoResult = await fetchImage(duckduckgoUrl);- buffer = duckduckgoResult.buffer;- contentType = duckduckgoResult.contentType;- status = duckduckgoResult.status;- imageUrl = duckduckgoUrl;-- logger.info(- {- status,- bufferLength: buffer.length,- contentType,- },- 'DuckDuckGo favicon result'- );- }-- // Accept any response as long as we have valid image data- if (buffer.length === 0) {- return reply- .status(404)- .header('Content-Type', 'text/plain')- .send('Not found');- }-- // Process the image (resize to 30x30 PNG, or serve ICO as-is)- const processedBuffer = await processImage(- buffer,- imageUrl.toString(),- contentType- );-- logger.info(- {- originalUrl: url.toString(),- originalBufferLength: buffer.length,- processedBufferLength: processedBuffer.length,- },- 'Favicon processing result'- );-- // `processImage` either passed an ICO through untouched or rasterized to- // PNG, so the response type is derived from what we produced rather than- // from whatever the upstream server claimed.- const responseContentType =- processedBuffer === buffer ? 'image/x-icon' : 'image/png';-- // Cache the result with correct content type- await setToCacheBinary(cacheKey, processedBuffer, responseContentType);-- setImageSecurityHeaders(reply, responseContentType);- reply.header('Cache-Control', 'public, max-age=3600, immutable');- return reply.send(processedBuffer);- } catch (error: any) {- if (error instanceof BlockedUrlError) {- logger.warn(- { url: request.query.url, reason: error.message },- 'Blocked favicon fetch'- );- reply.header('Cache-Control', 'no-store');- return reply- .status(400)- .header('Content-Type', 'text/plain')- .send('Bad request');- }-- logger.error({ err: error, url: request.query.url }, 'Favicon fetch error');-- const message =- process.env.NODE_ENV === 'production'- ? 'Bad request'- : (error?.message ?? 'Error');- reply.header('Cache-Control', 'no-store');- return reply.status(400).send(message);- }-}--export async function clearFavicons(- request: FastifyRequest,- reply: FastifyReply-) {- const redis = getRedisCache();- const keys = await redis.keys('favicon:*');-- // Delete both the binary data and content-type keys- for (const key of keys) {- await redis.del(key);- await redis.del(`${key}:ctype`);- }-- return reply.status(200).send('OK');+ return sendImageAsset(+ reply,+ await getFaviconCore(request.query.url, request.log)+ );}-export async function clearOgImages(- request: FastifyRequest,+export async function getOgImage(+ request: FastifyRequest<{ Querystring: { url: string } }>,reply: FastifyReply) {- const redis = getRedisCache();- const keys = await redis.keys('og:*');-- // Delete both the binary data and content-type keys- for (const key of keys) {- await redis.del(key);- await redis.del(`${key}:ctype`);- }-- return reply.status(200).send('OK');+ return sendImageAsset(+ reply,+ await getOgImageCore(request.query.url, request.log)+ );}export async function ping(@@ -367,17 +61,7 @@ export async function ping(reply: FastifyReply) {try {- await ch.insert({- table: TABLE_NAMES.self_hosting,- values: [- {- domain: request.body.domain,- count: request.body.count,- created_at: formatClickhouseDate(new Date(), true),- },- ],- format: 'JSONEachRow',- });+ await insertPingRecord(request.body);reply.status(200).send({message: 'Success',count: request.body.count,@@ -391,137 +75,16 @@ export async function ping(}}-export async function stats(request: FastifyRequest, reply: FastifyReply) {- const res = await getCache('api:stats', 60 * 60, async () => {- const projects = await chQuery<{ project_id: string; count: number }>(- `SELECT project_id, count(*) as count from ${TABLE_NAMES.events} GROUP by project_id order by count()`- );- const last24h = await chQuery<{ count: number }>(- `SELECT count(*) as count from ${TABLE_NAMES.events} WHERE created_at > now() - interval '24 hours'`- );- return { projects, last24hCount: last24h[0]?.count || 0 };- });-- reply.status(200).send({- projectsCount: res.projects.length,- eventsCount: res.projects.reduce((acc, { count }) => acc + count, 0),- eventsLast24hCount: res.last24hCount,- });+export async function stats(_request: FastifyRequest, reply: FastifyReply) {+ reply.status(200).send(await getStats());}export async function getGeo(request: FastifyRequest, reply: FastifyReply) {- const { ip, header } = getClientIpFromHeaders(request.headers);- const others = await Promise.all(- DEFAULT_IP_HEADER_ORDER.map(async (header) => {- const { ip } = getClientIpFromHeaders(request.headers, header);- return {- header,- ip,- geo: await getGeoLocation(ip),- };- })- );-- if (!ip) {+ const report = await getGeoReport(request.headers);+ if (!report.ok) {return reply.status(400).send('Bad Request');}- const geo = await getGeoLocation(ip);- return reply.status(200).send({- selected: {- geo,- ip,- header,- },- ...others.reduce(- (acc, other) => {- acc[other.header] = other;- return acc;- },- {} as Record<string, { ip: string; header: string; geo: GeoLocation }>- ),- });-}--export async function getOgImage(- request: FastifyRequest<{- Querystring: {- url: string;- };- }>,- reply: FastifyReply-) {- try {- const url = validateUrl(request.query.url);- if (!url) {- return getFavicon(request, reply);- }- await assertPublicUrl(url);-- const cacheKey = createCacheKey(url.toString(), 'og');-- // Check cache first- const cached = await getFromCacheBinary(cacheKey);- if (cached) {- setImageSecurityHeaders(reply, cached.contentType);- reply.header('Cache-Control', 'public, max-age=604800, immutable');- return reply.send(cached.buffer);- }-- let imageUrl: URL;-- // If it's a direct image URL, use it directly- if (isDirectImage(url)) {- imageUrl = url;- } else {- // For website URLs, extract OG image from HTML- const meta = await parseUrlMeta(url.toString());- if (meta?.ogImage) {- imageUrl = new URL(meta.ogImage);- } else {- // No OG image found, return a fallback- return getFavicon(request, reply);- }- }-- // Fetch the image- const { buffer, status } = await fetchImage(imageUrl);-- if (status !== 200 || buffer.length === 0) {- return getFavicon(request, reply);- }-- // Rasterize to a 300px-wide PNG- const processedBuffer = await processOgImage(buffer, imageUrl.toString());-- // Cache the result- await setToCacheBinary(cacheKey, processedBuffer, 'image/png');-- setImageSecurityHeaders(reply, 'image/png');- reply.header('Cache-Control', 'public, max-age=3600, immutable');- return reply.send(processedBuffer);- } catch (error: any) {- if (error instanceof BlockedUrlError) {- logger.warn(- { url: request.query.url, reason: error.message },- 'Blocked OG image fetch'- );- reply.header('Cache-Control', 'no-store');- return reply- .status(400)- .header('Content-Type', 'text/plain')- .send('Bad request');- }-- logger.error(- { err: error, url: request.query.url },- 'OG image fetch error'- );-- const message =- process.env.NODE_ENV === 'production'- ? 'Bad request'- : (error?.message ?? 'Error');- reply.header('Cache-Control', 'no-store');- return reply.status(400).send(message);- }+ return reply+ .status(200)+ .send({ selected: report.selected, ...report.others });}diff --git a/apps/api/src/controllers/tools.controller.ts b/apps/api/src/controllers/tools.controller.tsindex 0399ef7f4..f7ca49195 100644--- a/apps/api/src/controllers/tools.controller.ts+++ b/apps/api/src/controllers/tools.controller.ts@@ -1,802 +1,37 @@-import * as dns from 'node:dns/promises';-import * as net from 'node:net';-import * as tls from 'node:tls';-import { getClientIpFromHeaders, getGeoLocation } from '@openpanel/core';-import * as cheerio from 'cheerio';+// Dissolved into @openpanel/core's tools module (M7-008): site-checker and+// ip-lookup moved to packages/core/src/modules/tools/src/{site-checker,+// ip-lookup}.ts. This file stays (DELEGATE PATTERN) — it is `tools.router.ts`'s+// Fastify glue, a thin wrapper around the core functions, same shape as+// `manage.controller.ts` (M6-002).+import { runIpLookup, runSiteCheck } from '@openpanel/core';import type { FastifyReply, FastifyRequest } from 'fastify';-import {- assertPublicHostname,- assertPublicUrl,- BlockedUrlError,- isBlockedIp,- safeFetch,-} from '@/utils/safe-fetch';--const TIMEOUT_MS = 10_000;-const MAX_REDIRECTS = 10;-const MAX_HTML_BYTES = 5_000_000;--interface RedirectHop {- url: string;- status: number;- responseTime: number;-}--interface DetailedTiming {- dns: number;- connect: number;- tls: number;- ttfb: number;- total: number;-}--interface SiteCheckResult {- url: string;- finalUrl: string;- timestamp: string;- seo: {- title: { value: string; length: number };- description: { value: string; length: number };- canonical: string | null;- h1: string[];- robotsMeta: string | null;- robotsTxtStatus: 'allowed' | 'blocked' | 'error';- hasSitemap: boolean;- };- social: {- og: {- title: string | null;- description: string | null;- image: string | null;- url: string | null;- type: string | null;- };- twitter: {- card: string | null;- title: string | null;- description: string | null;- image: string | null;- };- };- technical: {- statusCode: number;- redirectChain: RedirectHop[];- responseTime: DetailedTiming;- contentType: string;- pageSize: number;- server: string | null;- ssl: {- valid: boolean;- issuer: string;- expires: string;- } | null;- };- hosting: {- ip: string;- location: {- country: string;- countryName?: string;- city: string;- region: string | null;- timezone: string | null;- latitude: number | null;- longitude: number | null;- } | null;- isp: string | null;- asn: string | null;- organization: string | null;- cdn: string | null;- };- security: {- csp: string | null;- xFrameOptions: string | null;- xContentTypeOptions: string | null;- hsts: string | null;- score: number;- };-}--const rateLimitMap = new Map<string, { count: number; resetAt: number }>();-const SITE_CHECK_WINDOW = 60 * 1000;-const SITE_CHECK_MAX = 10;-const IP_LOOKUP_WINDOW = 60 * 1000;-const IP_LOOKUP_MAX = 20;--function checkRateLimit(key: string, windowMs: number, max: number): boolean {- const now = Date.now();- const record = rateLimitMap.get(key);-- if (!record || now > record.resetAt) {- rateLimitMap.set(key, { count: 1, resetAt: now + windowMs });- return true;- }-- if (record.count >= max) {- return false;- }-- record.count++;- return true;-}--function detectCDN(headers: Headers): string | null {- const server = headers.get('server')?.toLowerCase() || '';- const cfRay = headers.get('cf-ray');- const vercelId = headers.get('x-vercel-id');- const fastly = headers.get('fastly-request-id');- const cloudfront = headers.get('x-amz-cf-id');-- if (cfRay || server.includes('cloudflare')) {- return 'Cloudflare';- }- if (vercelId || server.includes('vercel')) {- return 'Vercel';- }- if (fastly || server.includes('fastly')) {- return 'Fastly';- }- if (cloudfront || server.includes('cloudfront')) {- return 'CloudFront';- }- if (server.includes('nginx')) {- return 'Nginx';- }- if (server.includes('apache')) {- return 'Apache';- }-- return null;-}--async function checkRobotsTxt(- baseUrl: string,- path: string-): Promise<'allowed' | 'blocked' | 'error'> {- try {- const robotsUrl = new URL('/robots.txt', baseUrl);-- const response = await safeFetch(robotsUrl, {- timeoutMs: 5000,- maxBytes: 1_000_000,- headers: {- 'User-Agent': 'OpenPanel-SiteChecker/1.0',- },- });-- if (response.status < 200 || response.status >= 300) {- return 'error';- }-- const text = response.body.toString('utf8');- const rules = text.split('\n').filter((line) => {- const trimmed = line.trim();- return trimmed && !trimmed.startsWith('#');- });-- for (const rule of rules) {- if (rule.toLowerCase().startsWith('disallow:')) {- const pattern = rule.substring(9).trim();- if (pattern && path.includes(pattern.replace('*', ''))) {- return 'blocked';- }- }- }-- return 'allowed';- } catch {- return 'error';- }-}--async function checkSitemap(baseUrl: string): Promise<boolean> {- try {- const sitemapUrl = new URL('/sitemap.xml', baseUrl);-- const response = await safeFetch(sitemapUrl, {- timeoutMs: 3000,- maxBytes: 1_000_000,- headers: {- 'User-Agent': 'OpenPanel-SiteChecker/1.0',- },- });-- return response.status >= 200 && response.status < 300;- } catch {- return false;- }-}--async function getSSLInfo(- hostname: string-): Promise<SiteCheckResult['technical']['ssl'] | null> {- // Raw TLS handshakes are as good a probe as an HTTP request, so the host has- // to clear the same check, and we connect to the address we validated.- let address: string;- try {- [address] = (await assertPublicHostname(hostname)) as [string];- } catch {- return null;- }-- return new Promise((resolve) => {- const socket = tls.connect(- {- host: address,- port: 443,- servername: hostname,- rejectUnauthorized: false,- },- () => {- const cert = socket.getPeerCertificate(true);- socket.destroy();-- if (!(cert && cert.valid_to)) {- resolve(null);- return;- }-- resolve({- valid: new Date(cert.valid_to) > new Date(),- issuer: cert.issuer?.CN || 'Unknown',- expires: cert.valid_to,- });- }- );-- socket.on('error', () => {- resolve(null);- });-- setTimeout(() => {- socket.destroy();- resolve(null);- }, 3000);- });-}--async function resolveHostname(hostname: string): Promise<string> {- try {- const addresses = await dns.resolve4(hostname);- return addresses[0] || '';- } catch {- return '';- }-}--interface IPInfo {- isp: string | null;- asn: string | null;- organization: string | null;-}--async function getIPInfo(ip: string): Promise<IPInfo> {- // Don't hand internal addresses to a third-party lookup service.- if (!ip || isBlockedIp(ip)) {- return { isp: null, asn: null, organization: null };- }-- try {- const controller = new AbortController();- const timeout = setTimeout(() => controller.abort(), 3000);-- const response = await fetch(- `https://ip-api.com/json/${ip}?fields=isp,as,org,query,status`,- { signal: controller.signal }- );-- clearTimeout(timeout);-- if (!response.ok) {- return { isp: null, asn: null, organization: null };- }-- const data = (await response.json()) as {- status?: string;- isp?: string;- as?: string;- org?: string;- };-- if (data.status === 'fail') {- return { isp: null, asn: null, organization: null };- }-- return {- isp: data.isp || null,- asn: data.as ? `AS${data.as.split(' ')[0]}` : null,- organization: data.org || null,- };- } catch {- return { isp: null, asn: null, organization: null };- }-}--async function measureConnectionTime(- hostname: string,- port: number-): Promise<{ connectTime: number; tlsTime: number }> {- // Without this check the timing measurement doubles as an internal port- // scanner, since open and closed ports are trivially distinguishable.- let address: string;- try {- [address] = (await assertPublicHostname(hostname)) as [string];- } catch {- return { connectTime: 0, tlsTime: 0 };- }-- return new Promise((resolve) => {- const start = Date.now();- let connectTime = 0;- let tlsTime = 0;-- const socket = net.createConnection(port, address, () => {- connectTime = Date.now() - start;-- if (port === 443) {- const tlsStart = Date.now();- const tlsSocket = tls.connect({- socket,- servername: hostname,- rejectUnauthorized: false,- });-- tlsSocket.on('secureConnect', () => {- tlsTime = Date.now() - tlsStart;- tlsSocket.destroy();- resolve({ connectTime, tlsTime });- });-- tlsSocket.on('error', () => {- tlsSocket.destroy();- resolve({ connectTime, tlsTime: 0 });- });- } else {- socket.destroy();- resolve({ connectTime, tlsTime: 0 });- }- });-- socket.on('error', () => {- socket.destroy();- resolve({ connectTime: Date.now() - start, tlsTime: 0 });- });-- setTimeout(() => {- socket.destroy();- resolve({ connectTime: Date.now() - start, tlsTime: 0 });- }, 5000);- });-}--async function fetchWithRedirects(- url: string,- maxRedirects: number = MAX_REDIRECTS-): Promise<{- finalUrl: string;- redirectChain: RedirectHop[];- html: string;- headers: Headers;- statusCode: number;- timing: DetailedTiming;-}> {- const redirectChain: RedirectHop[] = [];- let currentUrl = url;- let finalHeaders: Headers | null = null;- let finalStatusCode = 0;- let finalHtml = '';- const totalStartTime = Date.now();-- let dnsTime = 0;- let connectTime = 0;- let tlsTime = 0;- let ttfbTime = 0;-- try {- const urlObj = new URL(currentUrl);- const hostname = urlObj.hostname;- const port = urlObj.port- ? Number.parseInt(urlObj.port, 10)- : urlObj.protocol === 'https:'- ? 443- : 80;-- const dnsStart = Date.now();- await assertPublicHostname(hostname);- dnsTime = Date.now() - dnsStart;-- if (port === 443 || port === 80) {- const connTiming = await measureConnectionTime(hostname, port);- connectTime = connTiming.connectTime;- tlsTime = connTiming.tlsTime;- }- } catch {- // continue- }-- let firstRequestStartTime = 0;-- for (let i = 0; i < maxRedirects; i++) {- try {- const hopStartTime = Date.now();- if (i === 0) {- firstRequestStartTime = hopStartTime;- }-- // `followRedirects: false` keeps the hop-by-hop timing this endpoint- // reports, while still validating and pinning each hop individually —- // a redirect cannot hand us off to an internal address.- const response = await safeFetch(currentUrl, {- followRedirects: false,- timeoutMs: TIMEOUT_MS,- maxBytes: MAX_HTML_BYTES,- headers: {- 'User-Agent':- 'Mozilla/5.0 (compatible; OpenPanel-SiteChecker/1.0; +https://openpanel.dev)',- Accept:- 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',- 'Accept-Language': 'en-US,en;q=0.5',- },- });-- const hopResponseTime = Date.now() - hopStartTime;-- if (i === 0 && ttfbTime === 0) {- ttfbTime = Date.now() - firstRequestStartTime;- }-- if (i === 0) {- finalHeaders = response.headers;- finalStatusCode = response.status;- }-- redirectChain.push({- url: currentUrl,- status: response.status,- responseTime: hopResponseTime,- });-- if (response.status >= 300 && response.status < 400) {- const location = response.headers.get('location');- if (location) {- currentUrl = new URL(location, currentUrl).toString();- continue;- }- }-- finalHtml = response.body.toString('utf8');- finalHeaders = response.headers;- finalStatusCode = response.status;- break;- } catch (error) {- if (error instanceof Error && error.name === 'AbortError') {- throw new Error('Request timeout');- }- throw error;- }- }-- const totalTime = Date.now() - totalStartTime;-- if (ttfbTime === 0 || ttfbTime > totalTime) {- const estimatedBodyTime = Math.max(0, totalTime * 0.3);- ttfbTime = Math.max(50, totalTime - estimatedBodyTime);- }-- return {- finalUrl: currentUrl,- redirectChain,- html: finalHtml,- headers: finalHeaders || new Headers(),- statusCode: finalStatusCode,- timing: {- dns: dnsTime,- connect: connectTime,- tls: tlsTime,- ttfb: ttfbTime,- total: totalTime,- },- };-}--function calculateSecurityScore(security: SiteCheckResult['security']): number {- let score = 0;- if (security.csp) {- score += 25;- }- if (security.xFrameOptions) {- score += 15;- }- if (security.xContentTypeOptions) {- score += 15;- }- if (security.hsts) {- score += 25;- }- if (- security.xFrameOptions?.toLowerCase() === 'deny' ||- security.xFrameOptions?.toLowerCase() === 'sameorigin'- ) {- score += 10;- }- if (security.xContentTypeOptions?.toLowerCase() === 'nosniff') {- score += 10;- }- return Math.min(100, score);-}export async function siteChecker(request: FastifyRequest<{ Querystring: { url?: string } }>,reply: FastifyReply) {- const urlParam = request.query.url;-- if (!urlParam) {- return reply.status(400).send({ error: 'URL parameter is required' });- }-- const { ip } = getClientIpFromHeaders(request.headers);- if (ip && !checkRateLimit(`site:${ip}`, SITE_CHECK_WINDOW, SITE_CHECK_MAX)) {- return reply- .status(429)- .send({ error: 'Rate limit exceeded. Please try again later.' });- }-- let url: URL;- try {- url = new URL(urlParam);- } catch {- return reply.status(400).send({ error: 'Invalid URL' });- }-- if (!(url.protocol && url.protocol.startsWith('http'))) {- url = new URL(`https://${urlParam}`);- }-- // Reject non-public destinations before any outbound connection is made.- // Each hop and each auxiliary probe re-checks independently, since a- // hostname can resolve differently between this check and the connection.- try {- await assertPublicUrl(url);- } catch (error) {- if (error instanceof BlockedUrlError) {- return reply.status(400).send({ error: 'Invalid URL' });- }- throw error;- }-- try {- const { finalUrl, redirectChain, html, headers, statusCode, timing } =- await fetchWithRedirects(url.toString());-- const finalUrlObj = new URL(finalUrl);- const $ = cheerio.load(html);-- const title = $('title').first().text().trim();- const description =- $('meta[name="description"]').attr('content') ||- $('meta[property="og:description"]').attr('content') ||- '';- const canonical =- $('link[rel="canonical"]').attr('href') ||- $('meta[property="og:url"]').attr('content') ||- null;- const h1Tags = $('h1')- .map((_, el) => $(el).text().trim())- .get()- .filter(Boolean);- const robotsMeta =- $('meta[name="robots"]').attr('content') ||- $('meta[name="googlebot"]').attr('content') ||- null;-- const ogTitle = $('meta[property="og:title"]').attr('content') || null;- const ogDescription =- $('meta[property="og:description"]').attr('content') || null;- const ogImage = $('meta[property="og:image"]').attr('content') || null;- const ogUrl = $('meta[property="og:url"]').attr('content') || null;- const ogType = $('meta[property="og:type"]').attr('content') || null;-- const twitterCard = $('meta[name="twitter:card"]').attr('content') || null;- const twitterTitle =- $('meta[name="twitter:title"]').attr('content') || null;- const twitterDescription =- $('meta[name="twitter:description"]').attr('content') || null;- const twitterImage =- $('meta[name="twitter:image"]').attr('content') || null;-- const csp = headers.get('content-security-policy');- const xFrameOptions = headers.get('x-frame-options');- const xContentTypeOptions = headers.get('x-content-type-options');- const hsts = headers.get('strict-transport-security');-- const contentType = headers.get('content-type') || 'unknown';- const server = headers.get('server');- const pageSize = new Blob([html]).size;-- const serverIp = await resolveHostname(finalUrlObj.hostname);- const geo = serverIp ? await getGeoLocation(serverIp) : null;- const ipInfo = serverIp- ? await getIPInfo(serverIp)- : { isp: null, asn: null, organization: null };- const cdn = detectCDN(headers);-- const ssl = await getSSLInfo(finalUrlObj.hostname);-- const robotsTxtStatus = await checkRobotsTxt(- finalUrl.toString(),- finalUrlObj.pathname- );-- const hasSitemap = await checkSitemap(finalUrl.toString());-- const security = {- csp,- xFrameOptions,- xContentTypeOptions,- hsts,- score: 0,- };- security.score = calculateSecurityScore(security);-- const result: SiteCheckResult = {- url: url.toString(),- finalUrl,- timestamp: new Date().toISOString(),- seo: {- title: { value: title, length: title.length },- description: { value: description, length: description.length },- canonical: canonical ? new URL(canonical, finalUrl).toString() : null,- h1: h1Tags,- robotsMeta,- robotsTxtStatus,- hasSitemap,- },- social: {- og: {- title: ogTitle,- description: ogDescription,- image: ogImage ? new URL(ogImage, finalUrl).toString() : null,- url: ogUrl ? new URL(ogUrl, finalUrl).toString() : null,- type: ogType,- },- twitter: {- card: twitterCard,- title: twitterTitle,- description: twitterDescription,- image: twitterImage- ? new URL(twitterImage, finalUrl).toString()- : null,- },- },- technical: {- statusCode,- redirectChain,- responseTime: timing,- contentType,- pageSize,- server,- ssl,- },- hosting: {- ip: serverIp,- location: geo- ? {- country: geo.country || '',- city: geo.city || '',- region: geo.region || null,- timezone: null,- latitude: geo.latitude || null,- longitude: geo.longitude || null,- }- : null,- isp: ipInfo.isp,- asn: ipInfo.asn,- organization: ipInfo.organization,- cdn,- },- security,- };-- return reply.send(result);- } catch (error) {- request.log.error({ err: error }, 'Site checker error');- if (error instanceof BlockedUrlError) {- return reply.status(400).send({ error: 'Invalid URL' });- }-- return reply.status(500).send({- error: error instanceof Error ? error.message : 'Failed to analyze site',- });- }-}--function isPrivateIP(ip: string): boolean {- if (ip === '::1') {- return true;- }- if (ip.startsWith('::ffff:127.')) {- return true;- }- if (ip.startsWith('127.')) {- return true;- }- if (ip.startsWith('10.')) {- return true;+ const outcome = await runSiteCheck(+ request.query.url,+ request.headers,+ request.log+ );+ if (outcome.status !== 200) {+ return reply.status(outcome.status).send({ error: outcome.error });}- if (ip.startsWith('192.168.')) {- return true;- }- if (ip.startsWith('172.')) {- const parts = ip.split('.');- if (parts.length >= 2) {- const secondOctet = Number.parseInt(parts[1] || '0', 10);- if (secondOctet >= 16 && secondOctet <= 31) {- return true;- }- }- }- if (- ip.startsWith('fc00:') ||- ip.startsWith('fd00:') ||- ip.startsWith('fe80:')- ) {- return true;- }- return false;+ return reply.send(outcome.result);}export async function ipLookup(request: FastifyRequest<{ Querystring: { ip?: string } }>,reply: FastifyReply) {- const ipParam = request.query.ip;-- const { ip: clientIp } = getClientIpFromHeaders(request.headers);- if (- clientIp &&- !checkRateLimit(`ip:${clientIp}`, IP_LOOKUP_WINDOW, IP_LOOKUP_MAX)- ) {- return reply- .status(429)- .send({ error: 'Rate limit exceeded. Please try again later.' });- }-- const ipToLookup = ipParam ? ipParam.trim() : clientIp || '';-- if (!ipToLookup) {- return reply- .status(400)- .send({ error: 'No IP address provided or detected' });- }-- const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/;- const ipv6Regex = /^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$/;- if (!(ipv4Regex.test(ipToLookup) || ipv6Regex.test(ipToLookup))) {- return reply.status(400).send({ error: 'Invalid IP address format' });- }-- try {- const geo = await getGeoLocation(ipToLookup);- const isLocalhost = ipToLookup === '127.0.0.1' || ipToLookup === '::1';- const isPrivate = isPrivateIP(ipToLookup);-- return reply.send({- ip: ipToLookup,- location: {- country: geo.country,- city: geo.city,- region: geo.region,- latitude: geo.latitude,- longitude: geo.longitude,- },- isLocalhost,- isPrivate,- });- } catch (error) {- request.log.error({ err: error }, 'IP lookup error');- return reply.status(500).send({- error:- error instanceof Error ? error.message : 'Failed to lookup IP address',- });- }+ const outcome = await runIpLookup(+ request.query.ip,+ request.headers,+ request.log+ );+ if (outcome.status !== 200) {+ return reply.status(outcome.status).send({ error: outcome.error });+ }+ return reply.send(outcome.result);}diff --git a/apps/api/src/routes/misc.router.ts b/apps/api/src/routes/misc.router.tsindex db5ffba6a..5d4ce8b45 100644--- a/apps/api/src/routes/misc.router.ts+++ b/apps/api/src/routes/misc.router.ts@@ -1,6 +1,9 @@-import * as controller from '@/controllers/misc.controller';import type { FastifyPluginCallback } from 'fastify';+import * as controller from '@/controllers/misc.controller';+// `/og/clear` and `/favicon/clear` are removed, not ported (M7-008,+// ADR-015 entry #6: RULED + DEAD — `docs/ANSWERS.md` §1.4 confirms nothing+// depends on them).const miscRouter: FastifyPluginCallback = async (fastify) => {fastify.route({method: 'POST',@@ -26,18 +29,6 @@ const miscRouter: FastifyPluginCallback = async (fastify) => {handler: controller.getOgImage,});- fastify.route({- method: 'GET',- url: '/og/clear',- handler: controller.clearOgImages,- });-- fastify.route({- method: 'GET',- url: '/favicon/clear',- handler: controller.clearFavicons,- });-fastify.route({method: 'GET',url: '/geo',diff --git a/apps/api/tsdown.config.ts b/apps/api/tsdown.config.tsindex 86b00dd5f..e41b81168 100644--- a/apps/api/tsdown.config.ts+++ b/apps/api/tsdown.config.ts@@ -14,6 +14,10 @@ const options: Options = {'@aws-sdk/client-s3','@aws-sdk/client-sts','@google-cloud/storage',+ // native module: loads its platform binding (@img/sharp-*) via a+ // runtime require resolved from its own file location, which breaks if+ // rolldown inlines sharp's source into this bundle (M7-008)+ 'sharp',],sourcemap: true,platform: 'node',diff --git a/apps/worker/src/jobs/cron.ping.ts b/apps/worker/src/jobs/cron.ping.tsindex 547ce83ae..b9e7440d1 100644--- a/apps/worker/src/jobs/cron.ping.ts+++ b/apps/worker/src/jobs/cron.ping.ts@@ -1,31 +1,10 @@-import { TABLE_NAMES, chQuery } from '@openpanel/db';+// Dissolved into @openpanel/core's misc module (M7-008): the outbound+// telemetry sweep moved to+// packages/core/src/modules/misc/misc.service.ts#runPingCron. This file+// stays (DELEGATE PATTERN) — it is `cron.ts`'s dispatcher `ping` case, a thin+// wrapper around the core function, same shape as `cron.delete.ts` (M6-001).+import { runPingCron } from '@openpanel/core';export async function ping() {- if (process.env.DISABLE_PING) {- return;- }-- const [res] = await chQuery<{ count: number }>(- `SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`,- );-- if (typeof res?.count === 'number') {- const response = await fetch('https://api.openpanel.dev/misc/ping', {- method: 'POST',- headers: {- 'Content-Type': 'application/json',- },- body: JSON.stringify({- domain:- process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL,- count: res?.count,- }),- });-- if (response.ok) {- return await response.json();- }-- throw new Error('Failed to ping the server');- }+ return await runPingCron();}diff --git a/packages/core/package.json b/packages/core/package.jsonindex 5ff893c30..cd5ac6165 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -43,6 +43,7 @@"@trpc/server": "^11.17.0","arctic": "^2.3.0","bullmq": "6.3.3",+ "cheerio": "^1.0.0","csv-parse": "^6.1.0","date-fns": "^3.3.1","elysia": "1.4.30",@@ -56,12 +57,14 @@"prom-client": "^15.1.3","qrcode": "^1.5.4","ramda": "^0.29.1",+ "sharp": "^0.33.5","short-unique-id": "^5.0.3","sqlstring": "^2.3.3","superjson": "^1.13.3","ua-parser-js": "^2.0.6","undici": "^8.10.0","unzipper": "^0.12.3",+ "url-metadata": "^5.4.1","zod": "catalog:"},"devDependencies": {diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 1603f917d..0cb6e15c9 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1117,6 +1117,37 @@ export {scanLegacyInsights,sendWeeklyDigests,} from './modules/insight/insight.service';+// Ported from apps/api's misc controller + apps/worker's cron.ping.ts (M7-008)+// — apps/api's misc controller and the worker's ping cron job call these+// directly, the same way V1 reaches every other dissolved service here.+// `GET /misc/og/clear` and `/misc/favicon/clear` are NOT ported (ADR-015+// entry #6: RULED + DEAD).+export type {+ GeoReport,+ ImageAssetResult,+ PingRecord,+ StatsResult,+} from './modules/misc/misc.service';+export {+ getFavicon,+ getGeoReport,+ getOgImage,+ getStats,+ insertPingRecord,+ runPingCron,+} from './modules/misc/misc.service';+// New module (M7-008) — apps/api's tools controller calls these directly,+// the same way V1 reaches every other dissolved service here.+export type {+ IpLookupOutcome,+ IpLookupResult,+} from './modules/tools/src/ip-lookup';+export { runIpLookup } from './modules/tools/src/ip-lookup';+export type {+ SiteCheckOutcome,+ SiteCheckResult,+} from './modules/tools/src/site-checker';+export { runSiteCheck } from './modules/tools/src/site-checker';export { dashboardRoutes, opsRoutes, publicApiRoutes } from './rest.routes';// The RPC base is on the barrel because it is the seam `@openpanel/trpc`// builds its 28 routers on: ONE tRPC instance, mounted by V1's Fastifydiff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex f2aede20b..e8fdaa74a 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -80,6 +80,10 @@ test('the insights queue carries the insight module job', () => {// The insight module's cron fragment (insight.jobs.ts), spread into the ONE// cron queue (ADR-005: "no cron module"). legacyCompat.cron maps a legacy// job's `type` straight onto its name, so these three ids double as job names.+// `ping` (M7-008, misc.jobs.ts) is the twentieth and only conditional one —+// its scheduler is added by `startSchedulers` only when SELF_HOSTED &&+// production (jobs/schedulers.ts's `PING_SCHEDULE`), but the job HANDLER is+// always registered here so a scheduled run always has somewhere to land.test('the cron queue carries the insight module cron fragment', () => {expect(Object.keys(queues.cron.jobs).sort()).toEqual([@@ -89,6 +93,7 @@ test('the cron queue carries the insight module cron fragment', () => {'insightCleanup','insightsDaily','onboarding',+ 'ping','sessionReaper','sessionVacuum','weeklyDigest',diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsindex cbb990493..1242d9384 100644--- a/packages/core/src/jobs.registry.ts+++ b/packages/core/src/jobs.registry.ts@@ -21,6 +21,7 @@ import {insightCronJobs,insightQueueJobs,} from './modules/insight/insight.jobs';+import { miscCronJobs } from './modules/misc/misc.jobs';import { notificationQueueJobs } from './modules/notification/notification.jobs';import { onboardingCronJobs } from './modules/onboarding/onboarding.jobs';import { organizationCronJobs } from './modules/organization/organization.jobs';@@ -70,6 +71,7 @@ const cron = defineQueue('cron', {...organizationCronJobs,...onboardingCronJobs,...sessionCronJobs,+ ...miscCronJobs,},});diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.tsindex f2b57302e..954b40271 100644--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts@@ -43,6 +43,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.tsindex 03e6a68f5..8c63dcacb 100644--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts@@ -39,6 +39,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},@@ -144,6 +145,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', asyncprofile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],},requestId: 'req_1',job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'gscSync' },diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.tsindex 99320b565..a8f276c12 100644--- a/packages/core/src/modules/import/import.jobs.test.ts+++ b/packages/core/src/modules/import/import.jobs.test.ts@@ -39,6 +39,7 @@ function stubJobCtx(importService: Partial<ImportService>): JobCtx {profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.tsindex 1e1b7e6f5..a278b7266 100644--- a/packages/core/src/modules/insight/insight.jobs.test.ts+++ b/packages/core/src/modules/insight/insight.jobs.test.ts@@ -50,6 +50,7 @@ function stubJobCtx(profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},@@ -138,6 +139,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId deprofile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],},requestId: 'req_1',job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },diff --git a/packages/core/src/modules/misc/misc.jobs.ts b/packages/core/src/modules/misc/misc.jobs.tsnew file mode 100644index 000000000..4893bb092--- /dev/null+++ b/packages/core/src/modules/misc/misc.jobs.ts@@ -0,0 +1,22 @@+// Ported from apps/worker/src/jobs/cron.ping.ts + boot-cron.ts's `ping`+// schedule (M7-008).+//+// `ping` is this module's fragment of the ONE `cron` queue's jobs: declared+// here and spread into jobs.registry.ts, same shape as+// organization.jobs.ts's `delete` (M6-001). Its schedule is NOT spread into+// `CRON_SCHEDULES` here — `PING_SCHEDULE` in jobs/schedulers.ts already+// declares it, conditionally on `SELF_HOSTED && production`, because it is+// the one scheduler that isn't always-on.++import { z } from 'zod';+import { defineJob } from '../../jobs/define';++/** This module's fragment of the `cron` queue's jobs. */+export const miscCronJobs = {+ ping: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ await ctx.services.misc.runPingCron();+ },+ }),+};diff --git a/packages/core/src/modules/misc/misc.routes.ts b/packages/core/src/modules/misc/misc.routes.tsnew file mode 100644index 000000000..5a7f52299--- /dev/null+++ b/packages/core/src/modules/misc/misc.routes.ts@@ -0,0 +1,83 @@+// Ported from apps/api/src/routes/misc.router.ts ++// apps/api/src/controllers/misc.controller.ts (M7-008). V1's Fastify router+// stays the LIVE route (DELEGATE PATTERN) and calls the same misc.service.ts+// functions this file does.+//+// `GET /misc/og/clear` and `GET /misc/favicon/clear` are dropped, not ported+// (ADR-015 entry #6: RULED + DEAD — `docs/ANSWERS.md` §1.4 confirms nothing+// depends on them).+//+// NAMED GAP, same as every other module here: not yet reachable — main.ts+// does not mount `dashboardRoutes` until a real `AppDeps` exists (P3/P4/P8).++import { z } from 'zod';+import { defineRoutes } from '../../http/define';+import {+ getFavicon,+ getGeoReport,+ getOgImage,+ getStats,+ insertPingRecord,+} from './misc.service';++const TAGS = ['Misc'];++const zPingBody = z.object({+ domain: z.string(),+ count: z.number(),+});++const zImageQuery = z.object({ url: z.string().optional() });++export const miscRoutes = defineRoutes((app) =>+ app+ .post(+ '/misc/ping',+ async ({ body, ctx, status }) => {+ try {+ await insertPingRecord(body);+ return { message: 'Success', count: body.count, domain: body.domain };+ } catch (error) {+ ctx.logger.error({ err: error }, 'Failed to insert ping');+ return status(500, { error: 'Failed to insert ping' });+ }+ },+ { body: zPingBody, detail: { tags: TAGS } }+ )+ .get('/misc/stats', () => getStats(), { detail: { tags: TAGS } })+ .get(+ '/misc/favicon',+ async ({ query, ctx, set }) => {+ const result = await getFavicon(query.url, ctx.logger);+ set.status = result.status;+ for (const [name, value] of Object.entries(result.headers)) {+ set.headers[name] = value;+ }+ return result.status === 200 ? result.buffer : result.body;+ },+ { query: zImageQuery, detail: { tags: TAGS } }+ )+ .get(+ '/misc/og',+ async ({ query, ctx, set }) => {+ const result = await getOgImage(query.url, ctx.logger);+ set.status = result.status;+ for (const [name, value] of Object.entries(result.headers)) {+ set.headers[name] = value;+ }+ return result.status === 200 ? result.buffer : result.body;+ },+ { query: zImageQuery, detail: { tags: TAGS } }+ )+ .get(+ '/misc/geo',+ async ({ ctx, status }) => {+ const report = await getGeoReport(ctx.headers);+ if (!report.ok) {+ return status(400, 'Bad Request');+ }+ return { selected: report.selected, ...report.others };+ },+ { detail: { tags: TAGS } }+ )+);diff --git a/packages/core/src/modules/misc/misc.service.ts b/packages/core/src/modules/misc/misc.service.tsnew file mode 100644index 000000000..0896d927e--- /dev/null+++ b/packages/core/src/modules/misc/misc.service.ts@@ -0,0 +1,497 @@+// Ported from apps/api/src/controllers/misc.controller.ts ++// apps/worker/src/jobs/cron.ping.ts (M7-008). V1's Fastify controller and the+// worker's `cron.ping.ts` stay the LIVE code paths (DELEGATE PATTERN) and+// call the functions below verbatim — no new behaviour.+//+// `GET/POST /misc/og/clear` and `/misc/favicon/clear` are NOT ported: ADR-015+// entry #6 grades them RULED + DEAD (`docs/ANSWERS.md` §1.4 confirms no proxy+// depends on them), so this wave is where V1 drops them too.+//+// ClickHouse access is lazy (`loadCh()` below) and targets the narrow+// `@openpanel/db/src/clickhouse/client` submodule, not the package's full+// barrel — same reason organization.service.ts's/onboarding.service.ts's+// headers give: this module is pulled into the eager `@openpanel/core`+// barrel via `services.ts`, and a dynamic edge into the FULL `@openpanel/db`+// barrel from a second core module (alongside export.service.ts's own) is+// exactly the "two dynamic-import edges close one cycle" shape that panics+// apps/worker's rolldown build (`module_finalizers "no entry found for+// key"`) — see the controller's rolldown-dynamic-import-cycle-panic note.+//+// `getCache` (not `getRedisCache`) is lazy for the same reason+// subscription.service.ts's header gives: a static `import { getCache } from+// '@openpanel/redis'` here broke every core test that partially mocks+// `@openpanel/redis` with no `getCache` export (e.g.+// organization.service.test.ts, gsc.service.test.ts).++import crypto from 'node:crypto';+import { getRedisCache } from '@openpanel/redis';+import { type GeoLocation, getGeoLocation } from '../../clients/geo';+import type { Logger } from '../../logger';+import type { ServiceDeps } from '../../services';+import {+ DEFAULT_IP_HEADER_ORDER,+ getClientIpFromHeaders,+} from '../../shared/get-client-ip';+import {+ assertPublicUrl,+ BlockedUrlError,+ safeFetch,+} from '../../shared/safe-fetch';+import {+ ALLOWED_IMAGE_CONTENT_TYPES,+ normalizeContentType,+ processImage,+ processOgImage,+} from './src/image-proxy';+import { parseUrlMeta } from './src/parse-url-meta';++function loadCh() {+ return import('@openpanel/db/src/clickhouse/client');+}++function loadCache() {+ return import('@openpanel/redis').then((m) => m.getCache);+}++const CACHE_TTL_SECONDS = 60 * 60 * 24; // 24h — binary favicon/og cache+const MAX_FETCH_BYTES = 1_000_000; // 1MB cap+const FETCH_TIMEOUT_MS = 10_000;+const FAVICON_USER_AGENT =+ 'OpenPanel-FaviconProxy/1.0 (+https://openpanel.dev)';+const CACHED_ASSET_CACHE_CONTROL = 'public, max-age=604800, immutable'; // 7d+const FRESH_ASSET_CACHE_CONTROL = 'public, max-age=3600, immutable'; // 1h+const STATS_CACHE_TTL_SECONDS = 60 * 60;+const STATS_CACHE_KEY = 'api:stats';+const IMAGE_EXTENSIONS = ['svg', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'ico'];++export interface ImageAssetOk {+ status: 200;+ buffer: Buffer;+ headers: Record<string, string>;+}+export interface ImageAssetFailure {+ status: 400 | 404;+ body: string;+ headers: Record<string, string>;+}+export type ImageAssetResult = ImageAssetOk | ImageAssetFailure;++function createCacheKey(url: string, prefix = 'favicon'): string {+ const hash = crypto.createHash('sha256').update(url).digest('hex');+ // v3: entries written before the SVG/raw-passthrough fix could hold+ // attacker-controlled bytes with an attacker-chosen content type, so the+ // old namespace is abandoned rather than served from.+ return `${prefix}:v3:${hash}`;+}++/**+ * Shape check only. The destination is validated by `assertPublicUrl` /+ * `safeFetch` right before each outbound request, because a hostname can+ * resolve differently between validation and connection.+ */+function validateUrl(raw?: string): URL | null {+ try {+ if (!raw) {+ throw new Error('Missing ?url');+ }+ const url = new URL(raw);+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {+ throw new Error('Only http/https URLs are allowed');+ }+ return url;+ } catch {+ return null;+ }+}++async function getFromCacheBinary(+ key: string+): Promise<{ buffer: Buffer; contentType: string } | null> {+ const redis = getRedisCache();+ const [bufferBase64, contentType] = await Promise.all([+ redis.get(key),+ redis.get(`${key}:ctype`),+ ]);+ if (!(bufferBase64 && contentType)) {+ return null;+ }+ return { buffer: Buffer.from(bufferBase64, 'base64'), contentType };+}++async function setToCacheBinary(+ key: string,+ buffer: Buffer,+ contentType: string+): Promise<void> {+ const redis = getRedisCache();+ await Promise.all([+ redis.set(key, buffer.toString('base64'), 'EX', CACHE_TTL_SECONDS),+ redis.set(`${key}:ctype`, contentType, 'EX', CACHE_TTL_SECONDS),+ ]);+}++async function fetchImage(+ url: URL,+ logger?: Pick<Logger, 'debug' | 'warn'>+): Promise<{ buffer: Buffer; contentType: string; status: number }> {+ try {+ // `safeFetch` validates and pins every hop, so a redirect cannot be used+ // to reach an internal address after the initial URL checks out.+ const result = await safeFetch(url, {+ timeoutMs: FETCH_TIMEOUT_MS,+ maxBytes: MAX_FETCH_BYTES,+ headers: {+ 'user-agent': FAVICON_USER_AGENT,+ accept: 'image/*,*/*;q=0.8',+ },+ });++ if (result.status !== 200) {+ return {+ buffer: Buffer.alloc(0),+ contentType: 'text/plain',+ status: result.status,+ };+ }++ const contentType = normalizeContentType(+ result.headers.get('content-type')+ );+ if (!ALLOWED_IMAGE_CONTENT_TYPES.has(contentType)) {+ logger?.debug(+ { url: url.toString(), contentType },+ 'Refusing non-image response'+ );+ return {+ buffer: Buffer.alloc(0),+ contentType: 'text/plain',+ status: 415,+ };+ }++ return { buffer: result.body, contentType, status: 200 };+ } catch (error) {+ if (error instanceof BlockedUrlError) {+ logger?.warn(+ { url: url.toString(), reason: error.message },+ 'Blocked image fetch'+ );+ }+ return { buffer: Buffer.alloc(0), contentType: 'text/plain', status: 500 };+ }+}++function isDirectImage(url: URL): boolean {+ return (+ IMAGE_EXTENSIONS.some((ext) => url.pathname.endsWith(`.${ext}`)) ||+ url.toString().includes('googleusercontent.com')+ );+}++function imageHeaders(+ contentType: string,+ cacheControl: string+): Record<string, string> {+ return {+ 'content-type': contentType,+ 'x-content-type-options': 'nosniff',+ 'content-security-policy': "default-src 'none'; sandbox",+ 'cross-origin-resource-policy': 'cross-origin',+ 'cache-control': cacheControl,+ };+}++function textFailure(status: 400 | 404, body: string): ImageAssetFailure {+ return { status, body, headers: { 'content-type': 'text/plain' } };+}++function blockedOrErrorResult(error: unknown): ImageAssetFailure {+ if (error instanceof BlockedUrlError) {+ return {+ ...textFailure(400, 'Bad request'),+ headers: { 'content-type': 'text/plain', 'cache-control': 'no-store' },+ };+ }+ const message =+ process.env.NODE_ENV === 'production'+ ? 'Bad request'+ : ((error as Error)?.message ?? 'Error');+ return {+ ...textFailure(400, message),+ headers: { 'content-type': 'text/plain', 'cache-control': 'no-store' },+ };+}++export async function getFavicon(+ rawUrl: string | undefined,+ logger?: Pick<Logger, 'debug' | 'warn' | 'error'>+): Promise<ImageAssetResult> {+ const url = validateUrl(rawUrl);+ if (!url) {+ return textFailure(404, 'Not found');+ }++ try {+ // Check the destination before anything else, so a blocked host is not+ // leaked to the DuckDuckGo fallback below either.+ await assertPublicUrl(url);++ const cacheKey = createCacheKey(url.toString());+ const cached = await getFromCacheBinary(cacheKey);+ if (cached) {+ return {+ status: 200,+ buffer: cached.buffer,+ headers: imageHeaders(cached.contentType, CACHED_ASSET_CACHE_CONTROL),+ };+ }++ let imageUrl: URL;+ if (isDirectImage(url)) {+ imageUrl = url;+ } else {+ const meta = await parseUrlMeta(url.toString());+ imageUrl = meta?.favicon+ ? new URL(meta.favicon)+ : new URL(`${url.origin}/favicon.ico`);+ }++ let { buffer, contentType } = await fetchImage(imageUrl, logger);++ // If the direct favicon fetch failed and it's not from DuckDuckGo's+ // service, try DuckDuckGo's favicon service as a fallback+ if (buffer.length === 0 && !imageUrl.hostname.includes('duckduckgo.com')) {+ const duckduckgoUrl = new URL(+ `https://icons.duckduckgo.com/ip3/${url.hostname}.ico`+ );+ const duckduckgoResult = await fetchImage(duckduckgoUrl, logger);+ buffer = duckduckgoResult.buffer;+ contentType = duckduckgoResult.contentType;+ imageUrl = duckduckgoUrl;+ }++ if (buffer.length === 0) {+ return textFailure(404, 'Not found');+ }++ // Process the image (resize to 30x30 PNG, or serve ICO as-is)+ const processedBuffer = await processImage(+ buffer,+ imageUrl.toString(),+ contentType,+ logger+ );++ // `processImage` either passed an ICO through untouched or rasterized to+ // PNG, so the response type is derived from what we produced rather than+ // from whatever the upstream server claimed.+ const responseContentType =+ processedBuffer === buffer ? 'image/x-icon' : 'image/png';++ await setToCacheBinary(cacheKey, processedBuffer, responseContentType);++ return {+ status: 200,+ buffer: processedBuffer,+ headers: imageHeaders(responseContentType, FRESH_ASSET_CACHE_CONTROL),+ };+ } catch (error) {+ logger?.error?.({ err: error, url: rawUrl }, 'Favicon fetch error');+ return blockedOrErrorResult(error);+ }+}++export async function getOgImage(+ rawUrl: string | undefined,+ logger?: Pick<Logger, 'debug' | 'warn' | 'error'>+): Promise<ImageAssetResult> {+ const url = validateUrl(rawUrl);+ if (!url) {+ return getFavicon(rawUrl, logger);+ }++ try {+ await assertPublicUrl(url);++ const cacheKey = createCacheKey(url.toString(), 'og');+ const cached = await getFromCacheBinary(cacheKey);+ if (cached) {+ return {+ status: 200,+ buffer: cached.buffer,+ headers: imageHeaders(cached.contentType, CACHED_ASSET_CACHE_CONTROL),+ };+ }++ let imageUrl: URL;+ if (isDirectImage(url)) {+ imageUrl = url;+ } else {+ const meta = await parseUrlMeta(url.toString());+ if (!meta?.ogImage) {+ return getFavicon(rawUrl, logger);+ }+ imageUrl = new URL(meta.ogImage);+ }++ const { buffer, status } = await fetchImage(imageUrl, logger);+ if (status !== 200 || buffer.length === 0) {+ return getFavicon(rawUrl, logger);+ }++ // Rasterize to a 300px-wide PNG+ const processedBuffer = await processOgImage(+ buffer,+ imageUrl.toString(),+ logger+ );+ await setToCacheBinary(cacheKey, processedBuffer, 'image/png');++ return {+ status: 200,+ buffer: processedBuffer,+ headers: imageHeaders('image/png', FRESH_ASSET_CACHE_CONTROL),+ };+ } catch (error) {+ logger?.error?.({ err: error, url: rawUrl }, 'OG image fetch error');+ return blockedOrErrorResult(error);+ }+}++export interface StatsResult {+ projectsCount: number;+ eventsCount: number;+ eventsLast24hCount: number;+}++export async function getStats(): Promise<StatsResult> {+ const { chQuery, TABLE_NAMES } = await loadCh();+ const getCache = await loadCache();+ const res = await getCache(+ STATS_CACHE_KEY,+ STATS_CACHE_TTL_SECONDS,+ async () => {+ const projects = await chQuery<{ project_id: string; count: number }>(+ `SELECT project_id, count(*) as count from ${TABLE_NAMES.events} GROUP by project_id order by count()`+ );+ const last24h = await chQuery<{ count: number }>(+ `SELECT count(*) as count from ${TABLE_NAMES.events} WHERE created_at > now() - interval '24 hours'`+ );+ return { projects, last24hCount: last24h[0]?.count || 0 };+ }+ );++ return {+ projectsCount: res.projects.length,+ eventsCount: res.projects.reduce((acc, { count }) => acc + count, 0),+ eventsLast24hCount: res.last24hCount,+ };+}++export interface PingRecord {+ domain: string;+ count: number;+}++/** `POST /misc/ping` — records a self-hosted instance's telemetry ping. */+export async function insertPingRecord(record: PingRecord): Promise<void> {+ const { ch, TABLE_NAMES, formatClickhouseDate } = await loadCh();+ await ch.insert({+ table: TABLE_NAMES.self_hosting,+ values: [+ {+ domain: record.domain,+ count: record.count,+ created_at: formatClickhouseDate(new Date(), true),+ },+ ],+ format: 'JSONEachRow',+ });+}++export interface GeoReportEntry {+ ip: string;+ header: string;+ geo: GeoLocation;+}+export type GeoReport =+ | { ok: false }+ | {+ ok: true;+ selected: GeoReportEntry;+ others: Record<string, GeoReportEntry>;+ };++export async function getGeoReport(+ headers: Record<string, string | string[] | undefined> | Headers+): Promise<GeoReport> {+ const { ip, header } = getClientIpFromHeaders(headers);+ const others = await Promise.all(+ DEFAULT_IP_HEADER_ORDER.map(async (headerName) => {+ const { ip: otherIp } = getClientIpFromHeaders(headers, headerName);+ return {+ header: headerName,+ ip: otherIp,+ geo: await getGeoLocation(otherIp),+ };+ })+ );++ if (!ip) {+ return { ok: false };+ }++ return {+ ok: true,+ selected: { ip, header, geo: await getGeoLocation(ip) },+ others: others.reduce<Record<string, GeoReportEntry>>((acc, other) => {+ acc[other.header] = other;+ return acc;+ }, {}),+ };+}++/**+ * The outbound half of "ping": a self-hosted instance's own cron job,+ * reporting its event count back to `api.openpanel.dev/misc/ping` (the H+ * route above, on OUR side). Ported verbatim from+ * apps/worker/src/jobs/cron.ping.ts.+ */+export async function runPingCron(): Promise<unknown> {+ if (process.env.DISABLE_PING) {+ return;+ }++ const { chQuery, TABLE_NAMES } = await loadCh();+ const [res] = await chQuery<{ count: number }>(+ `SELECT COUNT(*) as count FROM ${TABLE_NAMES.events}`+ );++ if (typeof res?.count !== 'number') {+ return;+ }++ const response = await fetch('https://api.openpanel.dev/misc/ping', {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ domain:+ process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL,+ count: res.count,+ }),+ });++ if (response.ok) {+ return await response.json();+ }++ throw new Error('Failed to ping the server');+}++export interface MiscService {+ runPingCron(): Promise<unknown>;+}++export function createMiscService(_deps: ServiceDeps): MiscService {+ return { runPingCron };+}diff --git a/apps/api/src/utils/image-proxy.test.ts b/packages/core/src/modules/misc/src/image-proxy.test.tssimilarity index 92%rename from apps/api/src/utils/image-proxy.test.tsrename to packages/core/src/modules/misc/src/image-proxy.test.tsindex 69ff9cfe8..991b9fbce 100644--- a/apps/api/src/utils/image-proxy.test.ts+++ b/packages/core/src/modules/misc/src/image-proxy.test.ts@@ -1,5 +1,7 @@+// Ported from apps/api/src/utils/image-proxy.test.ts (M7-008).++import { describe, expect, it } from 'bun:test';import sharp from 'sharp';-import { describe, expect, it } from 'vitest';import {ALLOWED_IMAGE_CONTENT_TYPES,hasIcoMagicBytes,@@ -14,7 +16,7 @@ const maliciousSvg = Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64">' +'<rect width="64" height="64" fill="red"/>' +'<script>alert(document.domain)</script>' +- '</svg>',+ '</svg>');async function makePng() {@@ -35,7 +37,7 @@ describe('processImage', () => {const result = await processImage(maliciousSvg,'https://evil.example/x.svg',- 'image/svg+xml',+ 'image/svg+xml');expect(result.subarray(0, 4)).toEqual(PNG_MAGIC);@@ -48,7 +50,7 @@ describe('processImage', () => {const result = await processImage(maliciousSvg,'https://evil.example/favicon',- 'image/svg+xml',+ 'image/svg+xml');expect(result.subarray(0, 4)).toEqual(PNG_MAGIC);@@ -64,7 +66,7 @@ describe('processImage', () => {const result = await processImage(png,'https://evil.example/small.png',- 'image/png',+ 'image/png');expect(result.subarray(0, 4)).toEqual(PNG_MAGIC);@@ -77,7 +79,7 @@ describe('processImage', () => {// not reach the passthrough.expect(hasIcoMagicBytes(html)).toBe(false);await expect(- processImage(html, 'https://evil.example/x.ico', 'image/x-icon'),+ processImage(html, 'https://evil.example/x.ico', 'image/x-icon')).rejects.toThrow();});@@ -91,7 +93,7 @@ describe('processImage', () => {const result = await processImage(ico,'https://example.com/favicon.ico',- 'image/x-icon',+ 'image/x-icon');expect(result).toBe(ico);@@ -102,7 +104,7 @@ describe('processOgImage', () => {it('rasterizes SVG instead of serving it verbatim', async () => {const result = await processOgImage(maliciousSvg,- 'https://evil.example/x.svg',+ 'https://evil.example/x.svg');expect(result.subarray(0, 4)).toEqual(PNG_MAGIC);@@ -111,7 +113,7 @@ describe('processOgImage', () => {it('rasterizes small images rather than passing the bytes through', async () => {const png = await makePng();- expect(png.length).toBeLessThan(10000);+ expect(png.length).toBeLessThan(10_000);const result = await processOgImage(png, 'https://example.com/og.png');expect(result.subarray(0, 4)).toEqual(PNG_MAGIC);@@ -121,7 +123,7 @@ describe('processOgImage', () => {describe('content type handling', () => {it('strips parameters and lowercases', () => {expect(normalizeContentType('Image/SVG+XML; charset=utf-8')).toBe(- 'image/svg+xml',+ 'image/svg+xml');expect(normalizeContentType(null)).toBe('');});diff --git a/apps/api/src/utils/image-proxy.ts b/packages/core/src/modules/misc/src/image-proxy.tssimilarity index 64%rename from apps/api/src/utils/image-proxy.tsrename to packages/core/src/modules/misc/src/image-proxy.tsindex 6572a0bb8..3d2bc7bb3 100644--- a/apps/api/src/utils/image-proxy.ts+++ b/packages/core/src/modules/misc/src/image-proxy.ts@@ -1,21 +1,20 @@-import { logger } from '@/utils/logger';+// Ported verbatim from apps/api/src/utils/image-proxy.ts (M7-008).+//+// These bytes come from a third-party server and are then served from the API+// origin, which also serves the credentialed `/trpc` and `/oauth` endpoints.+// Anything returned verbatim is therefore attacker-controlled content on a+// trusted origin — an SVG passed through untouched executes its `<script>` as+// the victim. Everything Sharp can decode is rasterized to PNG, which drops+// active content along with every other non-pixel payload (GHSA-r7hx-q6f4-vj6h).+import sharp from 'sharp';+import type { Logger } from '../../../logger';-/**- * Image normalization for the favicon/OG proxy.- *- * These bytes come from a third-party server and are then served from the API- * origin, which also serves the credentialed `/trpc` and `/oauth` endpoints.- * Anything returned verbatim is therefore attacker-controlled content on a- * trusted origin — an SVG passed through untouched executes its `<script>` as- * the victim. Everything Sharp can decode is rasterized to PNG, which drops- * active content along with every other non-pixel payload.- */+const FAVICON_SIZE = 30;+const OG_IMAGE_WIDTH = 300;-/**- * Content types we are willing to hand back to a browser. Anything else is- * dropped rather than proxied.- */+/** Content types we are willing to hand back to a browser. Anything else is+ * dropped rather than proxied. */export const ALLOWED_IMAGE_CONTENT_TYPES = new Set(['image/png','image/jpeg',@@ -63,6 +62,7 @@ export async function processImage(buffer: Buffer,originalUrl?: string,contentType?: string,+ logger?: Pick<Logger, 'debug' | 'warn'>): Promise<Buffer> {// ICO is the only format Sharp cannot decode, so it is the only passthrough.// Everything else — SVG very much included — is rasterized.@@ -71,30 +71,23 @@ export async function processImage(isIcoFile(originalUrl, contentType) &&hasIcoMagicBytes(buffer)) {- logger.debug(+ logger?.debug({ originalUrl, bufferSize: buffer.length },- 'Serving ICO file directly',+ 'Serving ICO file directly');return buffer;}try {return await sharp(buffer)- .resize(30, 30, {- fit: 'cover',- })+ .resize(FAVICON_SIZE, FAVICON_SIZE, { fit: 'cover' }).png().toBuffer();} catch (error) {- logger.warn(- {- err: error,- originalUrl,- bufferSize: buffer.length,- },- 'Sharp failed to process image',+ logger?.warn(+ { err: error, originalUrl, bufferSize: buffer.length },+ 'Sharp failed to process image');-throw error;}}@@ -103,27 +96,20 @@ export async function processImage(export async function processOgImage(buffer: Buffer,originalUrl?: string,+ logger?: Pick<Logger, 'warn'>): Promise<Buffer> {// Always rasterize. Returning the upstream bytes verbatim would let an// attacker serve arbitrary content from the API origin.try {return await sharp(buffer)- .resize(300, null, {- fit: 'inside',- withoutEnlargement: true,- })+ .resize(OG_IMAGE_WIDTH, null, { fit: 'inside', withoutEnlargement: true }).png().toBuffer();} catch (error) {- logger.warn(- {- err: error,- originalUrl,- bufferSize: buffer.length,- },- 'Sharp failed to process OG image',+ logger?.warn(+ { err: error, originalUrl, bufferSize: buffer.length },+ 'Sharp failed to process OG image');-throw error;}}diff --git a/apps/api/src/utils/parseUrlMeta.ts b/packages/core/src/modules/misc/src/parse-url-meta.tssimilarity index 83%rename from apps/api/src/utils/parseUrlMeta.tsrename to packages/core/src/modules/misc/src/parse-url-meta.tsindex f8b9f3b9a..475e91bf8 100644--- a/apps/api/src/utils/parseUrlMeta.ts+++ b/packages/core/src/modules/misc/src/parse-url-meta.ts@@ -1,5 +1,22 @@+// Ported verbatim from apps/api/src/utils/parseUrlMeta.ts (M7-008).+import urlMetadata from 'url-metadata';+const URL_METADATA_TIMEOUT_MS = 500;++interface UrlMetaData {+ favicons: {+ rel: string;+ href: string;+ sizes: string;+ }[];+ 'og:image'?: string;+ 'og:image:url'?: string;+ 'og:image:secure_url'?: string;+ 'twitter:image'?: string;+ 'twitter:image:src'?: string;+}+function fallbackFavicon(url: string) {try {const hostname = new URL(url).hostname;@@ -12,20 +29,15 @@ function fallbackFavicon(url: string) {function findBestFavicon(favicons: UrlMetaData['favicons']) {const match = favicons- .sort((a, b) => {- return a.rel.length - b.rel.length;- })+ .sort((a, b) => a.rel.length - b.rel.length).find((favicon) =>favicon.rel === 'shortcut icon' ||favicon.rel === 'icon' ||- favicon.rel === 'apple-touch-icon',+ favicon.rel === 'apple-touch-icon');- if (match) {- return match.href;- }- return null;+ return match ? match.href : null;}function findBestOgImage(data: UrlMetaData): string | null {@@ -57,27 +69,13 @@ function transform(data: UrlMetaData, url: string) {};}-interface UrlMetaData {- favicons: {- rel: string;- href: string;- sizes: string;- }[];- 'og:image'?: string;- 'og:image:url'?: string;- 'og:image:secure_url'?: string;- 'twitter:image'?: string;- 'twitter:image:src'?: string;-}-export async function parseUrlMeta(url: string) {try {const metadata = (await urlMetadata(url, {- timeout: 500,+ timeout: URL_METADATA_TIMEOUT_MS,})) as UrlMetaData;- const data = transform(metadata, url);- return data;- } catch (err) {+ return transform(metadata, url);+ } catch {return {favicon: fallbackFavicon(url),ogImage: null,diff --git a/packages/core/src/modules/notification/notification.jobs.test.ts b/packages/core/src/modules/notification/notification.jobs.test.tsindex b28515759..c27995020 100644--- a/packages/core/src/modules/notification/notification.jobs.test.ts+++ b/packages/core/src/modules/notification/notification.jobs.test.ts@@ -39,6 +39,7 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},diff --git a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts b/packages/core/src/modules/onboarding/onboarding.jobs.test.tsindex 7dc578af7..45f9791ab 100644--- a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts+++ b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts@@ -42,6 +42,7 @@ function stubJobCtx(profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},diff --git a/packages/core/src/modules/organization/organization.jobs.test.ts b/packages/core/src/modules/organization/organization.jobs.test.tsindex b2921e786..afade1295 100644--- a/packages/core/src/modules/organization/organization.jobs.test.ts+++ b/packages/core/src/modules/organization/organization.jobs.test.ts@@ -45,6 +45,7 @@ function stubJobCtx(profile: {} as Services['profile'],group: {} as Services['group'],chart: {} as Services['chart'],+ misc: {} as Services['misc'],};return {db: {},diff --git a/packages/core/src/modules/tools/src/ip-lookup.ts b/packages/core/src/modules/tools/src/ip-lookup.tsnew file mode 100644index 000000000..1db5740c8--- /dev/null+++ b/packages/core/src/modules/tools/src/ip-lookup.ts@@ -0,0 +1,118 @@+// Ported verbatim from apps/api/src/controllers/tools.controller.ts's+// `ipLookup` (M7-008).++import { getGeoLocation } from '../../../clients/geo';+import type { Logger } from '../../../logger';+import { getClientIpFromHeaders } from '../../../shared/get-client-ip';+import { checkRateLimit } from './rate-limit';++const IP_LOOKUP_WINDOW_MS = 60 * 1000;+const IP_LOOKUP_MAX = 20;++const IPV4_REGEX = /^(\d{1,3}\.){3}\d{1,3}$/;+const IPV6_REGEX = /^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$/;++export interface IpLookupResult {+ ip: string;+ location: {+ country: string | undefined;+ city: string | undefined;+ region: string | undefined;+ latitude: number | undefined;+ longitude: number | undefined;+ };+ isLocalhost: boolean;+ isPrivate: boolean;+}++export type IpLookupOutcome =+ | { status: 200; result: IpLookupResult }+ | { status: 400 | 429 | 500; error: string };++function isPrivateIP(ip: string): boolean {+ if (ip === '::1') {+ return true;+ }+ if (ip.startsWith('::ffff:127.')) {+ return true;+ }+ if (ip.startsWith('127.')) {+ return true;+ }+ if (ip.startsWith('10.')) {+ return true;+ }+ if (ip.startsWith('192.168.')) {+ return true;+ }+ if (ip.startsWith('172.')) {+ const parts = ip.split('.');+ if (parts.length >= 2) {+ const secondOctet = Number.parseInt(parts[1] || '0', 10);+ if (secondOctet >= 16 && secondOctet <= 31) {+ return true;+ }+ }+ }+ if (+ ip.startsWith('fc00:') ||+ ip.startsWith('fd00:') ||+ ip.startsWith('fe80:')+ ) {+ return true;+ }+ return false;+}++export async function runIpLookup(+ ipParam: string | undefined,+ headers: Record<string, string | string[] | undefined> | Headers,+ logger?: Pick<Logger, 'error'>+): Promise<IpLookupOutcome> {+ const { ip: clientIp } = getClientIpFromHeaders(headers);+ if (+ clientIp &&+ !checkRateLimit(`ip:${clientIp}`, IP_LOOKUP_WINDOW_MS, IP_LOOKUP_MAX)+ ) {+ return {+ status: 429,+ error: 'Rate limit exceeded. Please try again later.',+ };+ }++ const ipToLookup = ipParam ? ipParam.trim() : clientIp || '';++ if (!ipToLookup) {+ return { status: 400, error: 'No IP address provided or detected' };+ }++ if (!(IPV4_REGEX.test(ipToLookup) || IPV6_REGEX.test(ipToLookup))) {+ return { status: 400, error: 'Invalid IP address format' };+ }++ try {+ const geo = await getGeoLocation(ipToLookup);+ return {+ status: 200,+ result: {+ ip: ipToLookup,+ location: {+ country: geo.country,+ city: geo.city,+ region: geo.region,+ latitude: geo.latitude,+ longitude: geo.longitude,+ },+ isLocalhost: ipToLookup === '127.0.0.1' || ipToLookup === '::1',+ isPrivate: isPrivateIP(ipToLookup),+ },+ };+ } catch (error) {+ logger?.error({ err: error }, 'IP lookup error');+ return {+ status: 500,+ error:+ error instanceof Error ? error.message : 'Failed to lookup IP address',+ };+ }+}diff --git a/packages/core/src/modules/tools/src/rate-limit.test.ts b/packages/core/src/modules/tools/src/rate-limit.test.tsnew file mode 100644index 000000000..dd188710c--- /dev/null+++ b/packages/core/src/modules/tools/src/rate-limit.test.ts@@ -0,0 +1,19 @@+import { describe, expect, it } from 'bun:test';+import { checkRateLimit } from './rate-limit';++describe('checkRateLimit', () => {+ it('allows requests under the limit and blocks the next one', () => {+ const key = `test:${crypto.randomUUID()}`;+ expect(checkRateLimit(key, 60_000, 2)).toBe(true);+ expect(checkRateLimit(key, 60_000, 2)).toBe(true);+ expect(checkRateLimit(key, 60_000, 2)).toBe(false);+ });++ it('tracks independent keys separately', () => {+ const a = `test:${crypto.randomUUID()}`;+ const b = `test:${crypto.randomUUID()}`;+ expect(checkRateLimit(a, 60_000, 1)).toBe(true);+ expect(checkRateLimit(a, 60_000, 1)).toBe(false);+ expect(checkRateLimit(b, 60_000, 1)).toBe(true);+ });+});diff --git a/packages/core/src/modules/tools/src/rate-limit.ts b/packages/core/src/modules/tools/src/rate-limit.tsnew file mode 100644index 000000000..3f52bbb9e--- /dev/null+++ b/packages/core/src/modules/tools/src/rate-limit.ts@@ -0,0 +1,26 @@+// Shared in-memory rate limiter for the two public tools endpoints. Ported+// verbatim from apps/api/src/controllers/tools.controller.ts (M7-008): one+// process-local map, keyed per endpoint by the caller.++const rateLimitMap = new Map<string, { count: number; resetAt: number }>();++export function checkRateLimit(+ key: string,+ windowMs: number,+ max: number+): boolean {+ const now = Date.now();+ const record = rateLimitMap.get(key);++ if (!record || now > record.resetAt) {+ rateLimitMap.set(key, { count: 1, resetAt: now + windowMs });+ return true;+ }++ if (record.count >= max) {+ return false;+ }++ record.count++;+ return true;+}diff --git a/packages/core/src/modules/tools/src/site-checker.ts b/packages/core/src/modules/tools/src/site-checker.tsnew file mode 100644index 000000000..debddb51e--- /dev/null+++ b/packages/core/src/modules/tools/src/site-checker.ts@@ -0,0 +1,701 @@+// Ported verbatim from apps/api/src/controllers/tools.controller.ts's+// `siteChecker` (M7-008).++import * as dns from 'node:dns/promises';+import * as net from 'node:net';+import * as tls from 'node:tls';+import * as cheerio from 'cheerio';+import { getGeoLocation } from '../../../clients/geo';+import type { Logger } from '../../../logger';+import { getClientIpFromHeaders } from '../../../shared/get-client-ip';+import {+ assertPublicHostname,+ assertPublicUrl,+ BlockedUrlError,+ isBlockedIp,+ safeFetch,+} from '../../../shared/safe-fetch';+import { checkRateLimit } from './rate-limit';++const TIMEOUT_MS = 10_000;+const MAX_REDIRECTS = 10;+const MAX_HTML_BYTES = 5_000_000;+const SITE_CHECK_WINDOW_MS = 60 * 1000;+const SITE_CHECK_MAX = 10;+const SSL_PORT = 443;+const HTTP_PORT = 80;++interface RedirectHop {+ url: string;+ status: number;+ responseTime: number;+}++interface DetailedTiming {+ dns: number;+ connect: number;+ tls: number;+ ttfb: number;+ total: number;+}++export interface SiteCheckResult {+ url: string;+ finalUrl: string;+ timestamp: string;+ seo: {+ title: { value: string; length: number };+ description: { value: string; length: number };+ canonical: string | null;+ h1: string[];+ robotsMeta: string | null;+ robotsTxtStatus: 'allowed' | 'blocked' | 'error';+ hasSitemap: boolean;+ };+ social: {+ og: {+ title: string | null;+ description: string | null;+ image: string | null;+ url: string | null;+ type: string | null;+ };+ twitter: {+ card: string | null;+ title: string | null;+ description: string | null;+ image: string | null;+ };+ };+ technical: {+ statusCode: number;+ redirectChain: RedirectHop[];+ responseTime: DetailedTiming;+ contentType: string;+ pageSize: number;+ server: string | null;+ ssl: {+ valid: boolean;+ issuer: string;+ expires: string;+ } | null;+ };+ hosting: {+ ip: string;+ location: {+ country: string;+ countryName?: string;+ city: string;+ region: string | null;+ timezone: string | null;+ latitude: number | null;+ longitude: number | null;+ } | null;+ isp: string | null;+ asn: string | null;+ organization: string | null;+ cdn: string | null;+ };+ security: {+ csp: string | null;+ xFrameOptions: string | null;+ xContentTypeOptions: string | null;+ hsts: string | null;+ score: number;+ };+}++export type SiteCheckOutcome =+ | { status: 200; result: SiteCheckResult }+ | { status: 400 | 429 | 500; error: string };++function detectCDN(headers: Headers): string | null {+ const server = headers.get('server')?.toLowerCase() || '';+ const cfRay = headers.get('cf-ray');+ const vercelId = headers.get('x-vercel-id');+ const fastly = headers.get('fastly-request-id');+ const cloudfront = headers.get('x-amz-cf-id');++ if (cfRay || server.includes('cloudflare')) {+ return 'Cloudflare';+ }+ if (vercelId || server.includes('vercel')) {+ return 'Vercel';+ }+ if (fastly || server.includes('fastly')) {+ return 'Fastly';+ }+ if (cloudfront || server.includes('cloudfront')) {+ return 'CloudFront';+ }+ if (server.includes('nginx')) {+ return 'Nginx';+ }+ if (server.includes('apache')) {+ return 'Apache';+ }++ return null;+}++async function checkRobotsTxt(+ baseUrl: string,+ path: string+): Promise<'allowed' | 'blocked' | 'error'> {+ try {+ const robotsUrl = new URL('/robots.txt', baseUrl);++ const response = await safeFetch(robotsUrl, {+ timeoutMs: 5000,+ maxBytes: 1_000_000,+ headers: { 'User-Agent': 'OpenPanel-SiteChecker/1.0' },+ });++ if (response.status < 200 || response.status >= 300) {+ return 'error';+ }++ const text = response.body.toString('utf8');+ const rules = text.split('\n').filter((line) => {+ const trimmed = line.trim();+ return trimmed && !trimmed.startsWith('#');+ });++ for (const rule of rules) {+ if (rule.toLowerCase().startsWith('disallow:')) {+ const pattern = rule.substring(9).trim();+ if (pattern && path.includes(pattern.replace('*', ''))) {+ return 'blocked';+ }+ }+ }++ return 'allowed';+ } catch {+ return 'error';+ }+}++async function checkSitemap(baseUrl: string): Promise<boolean> {+ try {+ const sitemapUrl = new URL('/sitemap.xml', baseUrl);++ const response = await safeFetch(sitemapUrl, {+ timeoutMs: 3000,+ maxBytes: 1_000_000,+ headers: { 'User-Agent': 'OpenPanel-SiteChecker/1.0' },+ });++ return response.status >= 200 && response.status < 300;+ } catch {+ return false;+ }+}++async function getSSLInfo(+ hostname: string+): Promise<SiteCheckResult['technical']['ssl'] | null> {+ // Raw TLS handshakes are as good a probe as an HTTP request, so the host has+ // to clear the same check, and we connect to the address we validated.+ let address: string;+ try {+ [address] = (await assertPublicHostname(hostname)) as [string];+ } catch {+ return null;+ }++ return new Promise((resolve) => {+ const socket = tls.connect(+ {+ host: address,+ port: SSL_PORT,+ servername: hostname,+ rejectUnauthorized: false,+ },+ () => {+ const cert = socket.getPeerCertificate(true);+ socket.destroy();++ if (!cert?.valid_to) {+ resolve(null);+ return;+ }++ resolve({+ valid: new Date(cert.valid_to) > new Date(),+ issuer: cert.issuer?.CN || 'Unknown',+ expires: cert.valid_to,+ });+ }+ );++ socket.on('error', () => resolve(null));+ setTimeout(() => {+ socket.destroy();+ resolve(null);+ }, 3000);+ });+}++async function resolveHostname(hostname: string): Promise<string> {+ try {+ const addresses = await dns.resolve4(hostname);+ return addresses[0] || '';+ } catch {+ return '';+ }+}++interface IPInfo {+ isp: string | null;+ asn: string | null;+ organization: string | null;+}++async function getIPInfo(ip: string): Promise<IPInfo> {+ // Don't hand internal addresses to a third-party lookup service.+ if (!ip || isBlockedIp(ip)) {+ return { isp: null, asn: null, organization: null };+ }++ try {+ const controller = new AbortController();+ const timeout = setTimeout(() => controller.abort(), 3000);++ const response = await fetch(+ `https://ip-api.com/json/${ip}?fields=isp,as,org,query,status`,+ { signal: controller.signal }+ );++ clearTimeout(timeout);++ if (!response.ok) {+ return { isp: null, asn: null, organization: null };+ }++ const data = (await response.json()) as {+ status?: string;+ isp?: string;+ as?: string;+ org?: string;+ };++ if (data.status === 'fail') {+ return { isp: null, asn: null, organization: null };+ }++ return {+ isp: data.isp || null,+ asn: data.as ? `AS${data.as.split(' ')[0]}` : null,+ organization: data.org || null,+ };+ } catch {+ return { isp: null, asn: null, organization: null };+ }+}++async function measureConnectionTime(+ hostname: string,+ port: number+): Promise<{ connectTime: number; tlsTime: number }> {+ // Without this check the timing measurement doubles as an internal port+ // scanner, since open and closed ports are trivially distinguishable.+ let address: string;+ try {+ [address] = (await assertPublicHostname(hostname)) as [string];+ } catch {+ return { connectTime: 0, tlsTime: 0 };+ }++ return new Promise((resolve) => {+ const start = Date.now();+ let connectTime = 0;+ let tlsTime = 0;++ const socket = net.createConnection(port, address, () => {+ connectTime = Date.now() - start;++ if (port === SSL_PORT) {+ const tlsStart = Date.now();+ const tlsSocket = tls.connect({+ socket,+ servername: hostname,+ rejectUnauthorized: false,+ });++ tlsSocket.on('secureConnect', () => {+ tlsTime = Date.now() - tlsStart;+ tlsSocket.destroy();+ resolve({ connectTime, tlsTime });+ });++ tlsSocket.on('error', () => {+ tlsSocket.destroy();+ resolve({ connectTime, tlsTime: 0 });+ });+ } else {+ socket.destroy();+ resolve({ connectTime, tlsTime: 0 });+ }+ });++ socket.on('error', () => {+ socket.destroy();+ resolve({ connectTime: Date.now() - start, tlsTime: 0 });+ });++ setTimeout(() => {+ socket.destroy();+ resolve({ connectTime: Date.now() - start, tlsTime: 0 });+ }, 5000);+ });+}++async function fetchWithRedirects(+ url: string,+ maxRedirects: number = MAX_REDIRECTS+): Promise<{+ finalUrl: string;+ redirectChain: RedirectHop[];+ html: string;+ headers: Headers;+ statusCode: number;+ timing: DetailedTiming;+}> {+ const redirectChain: RedirectHop[] = [];+ let currentUrl = url;+ let finalHeaders: Headers | null = null;+ let finalStatusCode = 0;+ let finalHtml = '';+ const totalStartTime = Date.now();++ let dnsTime = 0;+ let connectTime = 0;+ let tlsTime = 0;+ let ttfbTime = 0;++ try {+ const urlObj = new URL(currentUrl);+ const hostname = urlObj.hostname;+ const port = urlObj.port+ ? Number.parseInt(urlObj.port, 10)+ : urlObj.protocol === 'https:'+ ? SSL_PORT+ : HTTP_PORT;++ const dnsStart = Date.now();+ await assertPublicHostname(hostname);+ dnsTime = Date.now() - dnsStart;++ if (port === SSL_PORT || port === HTTP_PORT) {+ const connTiming = await measureConnectionTime(hostname, port);+ connectTime = connTiming.connectTime;+ tlsTime = connTiming.tlsTime;+ }+ } catch {+ // continue+ }++ let firstRequestStartTime = 0;++ for (let i = 0; i < maxRedirects; i++) {+ try {+ const hopStartTime = Date.now();+ if (i === 0) {+ firstRequestStartTime = hopStartTime;+ }++ // `followRedirects: false` keeps the hop-by-hop timing this endpoint+ // reports, while still validating and pinning each hop individually —+ // a redirect cannot hand us off to an internal address.+ const response = await safeFetch(currentUrl, {+ followRedirects: false,+ timeoutMs: TIMEOUT_MS,+ maxBytes: MAX_HTML_BYTES,+ headers: {+ 'User-Agent':+ 'Mozilla/5.0 (compatible; OpenPanel-SiteChecker/1.0; +https://openpanel.dev)',+ Accept:+ 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',+ 'Accept-Language': 'en-US,en;q=0.5',+ },+ });++ const hopResponseTime = Date.now() - hopStartTime;++ if (i === 0 && ttfbTime === 0) {+ ttfbTime = Date.now() - firstRequestStartTime;+ }++ if (i === 0) {+ finalHeaders = response.headers;+ finalStatusCode = response.status;+ }++ redirectChain.push({+ url: currentUrl,+ status: response.status,+ responseTime: hopResponseTime,+ });++ if (response.status >= 300 && response.status < 400) {+ const location = response.headers.get('location');+ if (location) {+ currentUrl = new URL(location, currentUrl).toString();+ continue;+ }+ }++ finalHtml = response.body.toString('utf8');+ finalHeaders = response.headers;+ finalStatusCode = response.status;+ break;+ } catch (error) {+ if (error instanceof Error && error.name === 'AbortError') {+ throw new Error('Request timeout');+ }+ throw error;+ }+ }++ const totalTime = Date.now() - totalStartTime;++ if (ttfbTime === 0 || ttfbTime > totalTime) {+ const estimatedBodyTime = Math.max(0, totalTime * 0.3);+ ttfbTime = Math.max(50, totalTime - estimatedBodyTime);+ }++ return {+ finalUrl: currentUrl,+ redirectChain,+ html: finalHtml,+ headers: finalHeaders || new Headers(),+ statusCode: finalStatusCode,+ timing: {+ dns: dnsTime,+ connect: connectTime,+ tls: tlsTime,+ ttfb: ttfbTime,+ total: totalTime,+ },+ };+}++function calculateSecurityScore(security: SiteCheckResult['security']): number {+ let score = 0;+ if (security.csp) {+ score += 25;+ }+ if (security.xFrameOptions) {+ score += 15;+ }+ if (security.xContentTypeOptions) {+ score += 15;+ }+ if (security.hsts) {+ score += 25;+ }+ if (+ security.xFrameOptions?.toLowerCase() === 'deny' ||+ security.xFrameOptions?.toLowerCase() === 'sameorigin'+ ) {+ score += 10;+ }+ if (security.xContentTypeOptions?.toLowerCase() === 'nosniff') {+ score += 10;+ }+ return Math.min(100, score);+}++export async function runSiteCheck(+ urlParam: string | undefined,+ headers: Record<string, string | string[] | undefined> | Headers,+ logger?: Pick<Logger, 'error'>+): Promise<SiteCheckOutcome> {+ if (!urlParam) {+ return { status: 400, error: 'URL parameter is required' };+ }++ const { ip } = getClientIpFromHeaders(headers);+ if (+ ip &&+ !checkRateLimit(`site:${ip}`, SITE_CHECK_WINDOW_MS, SITE_CHECK_MAX)+ ) {+ return {+ status: 429,+ error: 'Rate limit exceeded. Please try again later.',+ };+ }++ let url: URL;+ try {+ url = new URL(urlParam);+ } catch {+ return { status: 400, error: 'Invalid URL' };+ }++ if (!url.protocol?.startsWith('http')) {+ url = new URL(`https://${urlParam}`);+ }++ // Reject non-public destinations before any outbound connection is made.+ // Each hop and each auxiliary probe re-checks independently, since a+ // hostname can resolve differently between this check and the connection.+ try {+ await assertPublicUrl(url);+ } catch (error) {+ if (error instanceof BlockedUrlError) {+ return { status: 400, error: 'Invalid URL' };+ }+ throw error;+ }++ try {+ const {+ finalUrl,+ redirectChain,+ html,+ headers: responseHeaders,+ statusCode,+ timing,+ } = await fetchWithRedirects(url.toString());++ const finalUrlObj = new URL(finalUrl);+ const $ = cheerio.load(html);++ const title = $('title').first().text().trim();+ const description =+ $('meta[name="description"]').attr('content') ||+ $('meta[property="og:description"]').attr('content') ||+ '';+ const canonical =+ $('link[rel="canonical"]').attr('href') ||+ $('meta[property="og:url"]').attr('content') ||+ null;+ const h1Tags = $('h1')+ .map((_, el) => $(el).text().trim())+ .get()+ .filter(Boolean);+ const robotsMeta =+ $('meta[name="robots"]').attr('content') ||+ $('meta[name="googlebot"]').attr('content') ||+ null;++ const ogTitle = $('meta[property="og:title"]').attr('content') || null;+ const ogDescription =+ $('meta[property="og:description"]').attr('content') || null;+ const ogImage = $('meta[property="og:image"]').attr('content') || null;+ const ogUrl = $('meta[property="og:url"]').attr('content') || null;+ const ogType = $('meta[property="og:type"]').attr('content') || null;++ const twitterCard = $('meta[name="twitter:card"]').attr('content') || null;+ const twitterTitle =+ $('meta[name="twitter:title"]').attr('content') || null;+ const twitterDescription =+ $('meta[name="twitter:description"]').attr('content') || null;+ const twitterImage =+ $('meta[name="twitter:image"]').attr('content') || null;++ const csp = responseHeaders.get('content-security-policy');+ const xFrameOptions = responseHeaders.get('x-frame-options');+ const xContentTypeOptions = responseHeaders.get('x-content-type-options');+ const hsts = responseHeaders.get('strict-transport-security');++ const contentType = responseHeaders.get('content-type') || 'unknown';+ const server = responseHeaders.get('server');+ const pageSize = new Blob([html]).size;++ const serverIp = await resolveHostname(finalUrlObj.hostname);+ const geo = serverIp ? await getGeoLocation(serverIp) : null;+ const ipInfo = serverIp+ ? await getIPInfo(serverIp)+ : { isp: null, asn: null, organization: null };+ const cdn = detectCDN(responseHeaders);++ const ssl = await getSSLInfo(finalUrlObj.hostname);+ const robotsTxtStatus = await checkRobotsTxt(+ finalUrl.toString(),+ finalUrlObj.pathname+ );+ const hasSitemap = await checkSitemap(finalUrl.toString());++ const security = {+ csp,+ xFrameOptions,+ xContentTypeOptions,+ hsts,+ score: 0,+ };+ security.score = calculateSecurityScore(security);++ const result: SiteCheckResult = {+ url: url.toString(),+ finalUrl,+ timestamp: new Date().toISOString(),+ seo: {+ title: { value: title, length: title.length },+ description: { value: description, length: description.length },+ canonical: canonical ? new URL(canonical, finalUrl).toString() : null,+ h1: h1Tags,+ robotsMeta,+ robotsTxtStatus,+ hasSitemap,+ },+ social: {+ og: {+ title: ogTitle,+ description: ogDescription,+ image: ogImage ? new URL(ogImage, finalUrl).toString() : null,+ url: ogUrl ? new URL(ogUrl, finalUrl).toString() : null,+ type: ogType,+ },+ twitter: {+ card: twitterCard,+ title: twitterTitle,+ description: twitterDescription,+ image: twitterImage+ ? new URL(twitterImage, finalUrl).toString()+ : null,+ },+ },+ technical: {+ statusCode,+ redirectChain,+ responseTime: timing,+ contentType,+ pageSize,+ server,+ ssl,+ },+ hosting: {+ ip: serverIp,+ location: geo+ ? {+ country: geo.country || '',+ city: geo.city || '',+ region: geo.region || null,+ timezone: null,+ latitude: geo.latitude || null,+ longitude: geo.longitude || null,+ }+ : null,+ isp: ipInfo.isp,+ asn: ipInfo.asn,+ organization: ipInfo.organization,+ cdn,+ },+ security,+ };++ return { status: 200, result };+ } catch (error) {+ if (error instanceof BlockedUrlError) {+ return { status: 400, error: 'Invalid URL' };+ }+ logger?.error({ err: error }, 'Site checker error');+ return {+ status: 500,+ error: error instanceof Error ? error.message : 'Failed to analyze site',+ };+ }+}diff --git a/packages/core/src/modules/tools/tools.routes.ts b/packages/core/src/modules/tools/tools.routes.tsnew file mode 100644index 000000000..b66d3e0d6--- /dev/null+++ b/packages/core/src/modules/tools/tools.routes.ts@@ -0,0 +1,48 @@+// Ported from apps/api/src/routes/tools.router.ts ++// apps/api/src/controllers/tools.controller.ts (M7-008). V1's Fastify router+// stays the LIVE route (DELEGATE PATTERN) and calls the same+// src/site-checker.ts + src/ip-lookup.ts functions this file does. Both+// routes are hidden from the OpenAPI document, matching V1's+// `schema: { hide: true }`.+//+// NAMED GAP, same as every other module here: not yet reachable — main.ts+// does not mount `publicApiRoutes` until a real `AppDeps` exists (P3/P4/P8).++import { z } from 'zod';+import { defineRoutes } from '../../http/define';+import { runIpLookup } from './src/ip-lookup';+import { runSiteCheck } from './src/site-checker';++const TAGS = ['Tools'];++export const toolsRoutes = defineRoutes((app) =>+ app+ .get(+ '/site-checker',+ async ({ query, ctx, status }) => {+ const outcome = await runSiteCheck(query.url, ctx.headers, ctx.logger);+ if (outcome.status !== 200) {+ return status(outcome.status, { error: outcome.error });+ }+ return outcome.result;+ },+ {+ query: z.object({ url: z.string().optional() }),+ detail: { tags: TAGS, hide: true },+ }+ )+ .get(+ '/ip-lookup',+ async ({ query, ctx, status }) => {+ const outcome = await runIpLookup(query.ip, ctx.headers, ctx.logger);+ if (outcome.status !== 200) {+ return status(outcome.status, { error: outcome.error });+ }+ return outcome.result;+ },+ {+ query: z.object({ ip: z.string().optional() }),+ detail: { tags: TAGS, hide: true },+ }+ )+);diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.tsindex c3f9c05dd..bd5ea742c 100644--- a/packages/core/src/rest.routes.ts+++ b/packages/core/src/rest.routes.ts@@ -17,10 +17,12 @@ import { healthRoutes } from './modules/health/health.routes';import { importRoutes } from './modules/import/import.routes';import { integrationRoutes } from './modules/integration/integration.routes';import { mcpRoutes } from './modules/mcp/mcp.routes';+import { miscRoutes } from './modules/misc/misc.routes';import { profileRoutes } from './modules/profile/profile.routes';import { projectRoutes } from './modules/project/project.routes';import { realtimeRoutes } from './modules/realtime/realtime.routes';import { subscriptionRoutes } from './modules/subscription/subscription.routes';+import { toolsRoutes } from './modules/tools/tools.routes';// ADR-002/ADR-003 pin: @elysiajs/openapi at the 1.4.15 fallback (2.0 is// NO-GO per spike 6). `specPath` is set explicitly rather than taking the@@ -44,6 +46,8 @@ const OPENAPI_EXCLUDED_PATHS = ['/metrics'];// None of it is yet reachable: the clientAuth macro's authenticator is a P8// stub that always returns null (import.routes.ts's header), and main.ts// does not mount `publicApiRoutes` until a real `AppDeps` exists (P3/P4/P8).+// tools' `/site-checker` + `/ip-lookup` (M7-008) joined it too — unauthenticated+// like V1's, per the module map.export const publicApiRoutes = (deps: AppDeps) =>new Elysia({ name: 'core/public-api-routes' }).use(importRoutes(deps))@@ -51,7 +55,8 @@ export const publicApiRoutes = (deps: AppDeps) =>.use(clientRoutes(deps)).use(profileRoutes(deps)).use(exportRoutes(deps))- .use(insightsRoutes(deps));+ .use(insightsRoutes(deps))+ .use(toolsRoutes(deps));// gsc's callback was the first module to land here (M5-002); assistant's// `/ai/agents/*` (M5-005), mcp's `/mcp` (M5-007) and auth's `/oauth/*`@@ -69,6 +74,7 @@ export const publicApiRoutes = (deps: AppDeps) =>// `/oauth`'s.// realtime's `/live/*` websockets (M6-007) joined it too — `/live` IS in// `corsPaths`, same as `/trpc`/`/webhook`/`/oauth`/`/misc`/`/ai`.+// misc's `/misc/*` (M7-008) joined it too — `/misc` IS in `corsPaths`.export const dashboardRoutes = (deps: AppDeps) =>new Elysia({ name: 'core/dashboard-routes' }).use(gscRoutes(deps))@@ -77,7 +83,8 @@ export const dashboardRoutes = (deps: AppDeps) =>.use(authRoutes(deps)).use(integrationRoutes(deps)).use(subscriptionRoutes(deps))- .use(realtimeRoutes(deps));+ .use(realtimeRoutes(deps))+ .use(miscRoutes(deps));// healthz/metrics/misc share V1's ops surface (http/context.ts's// UNLOGGED_PATH_PREFIXES) — unauthenticated, uncorsed, unlogged. /metrics anddiff --git a/packages/core/src/services.ts b/packages/core/src/services.tsindex 19c555008..6ffaf85fa 100644--- a/packages/core/src/services.ts+++ b/packages/core/src/services.ts@@ -32,6 +32,10 @@ import {createInsightService,type InsightService,} from './modules/insight/insight.service';+import {+ createMiscService,+ type MiscService,+} from './modules/misc/misc.service';import {createNotificationService,type NotificationService,@@ -82,6 +86,7 @@ export interface Services {profile: ProfileService;group: GroupService;chart: ChartService;+ misc: MiscService;}export function createServices(deps: ServiceDeps): Services {@@ -102,5 +107,6 @@ export function createServices(deps: ServiceDeps): Services {profile: createProfileService(deps),group: createGroupService(deps),chart: createChartService(deps),+ misc: createMiscService(deps),};}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 3bc85e5b2..855c84aa2 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -160,9 +160,6 @@ importers:'@trpc/server':specifier: ^11.17.0version: 11.17.0([email protected])- cheerio:- specifier: ^1.0.0- version: 1.1.2fast-json-stable-hash:specifier: ^1.0.3version: 1.0.3@@ -196,9 +193,6 @@ importers:source-map-support:specifier: ^0.5.21version: 0.5.21- url-metadata:- specifier: ^5.4.1- version: 5.4.1uuid:specifier: ^9.0.1version: 9.0.1@@ -1157,6 +1151,9 @@ importers:bullmq:specifier: 6.3.3version: 6.3.3([email protected]([email protected]))+ cheerio:+ specifier: ^1.0.0+ version: 1.1.2csv-parse:specifier: ^6.1.0version: 6.1.0@@ -1196,6 +1193,9 @@ importers:ramda:specifier: ^0.29.1version: 0.29.1+ sharp:+ specifier: ^0.33.5+ version: 0.33.5short-unique-id:specifier: ^5.0.3version: 5.0.3@@ -1214,6 +1214,9 @@ importers:unzipper:specifier: ^0.12.3version: 0.12.5+ url-metadata:+ specifier: ^5.4.1+ version: 5.4.1zod:specifier: 'catalog:'version: 4.3.6@@ -11751,14 +11754,6 @@ packages:engines: {node: '>=0.10'}hasBin: true- resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}- engines: {node: '>=8'}-- resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==}- engines: {node: '>=8'}-resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}engines: {node: '>=8'}@@ -17035,11 +17030,6 @@ packages:engines: {node: '>=10'}hasBin: true- resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==}- engines: {node: '>=10'}- hasBin: true-resolution: {integrity: sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==}engines: {node: '>=10'}@@ -23955,7 +23945,7 @@ snapshots:'@img/[email protected]':dependencies:- '@emnapi/runtime': 1.3.1+ '@emnapi/runtime': 1.7.1optional: true'@img/[email protected]':@@ -24134,7 +24124,7 @@ snapshots:'@mapbox/[email protected]([email protected])':dependencies:consola: 3.4.2- detect-libc: 2.0.4+ detect-libc: 2.1.2https-proxy-agent: 7.0.6([email protected])node-fetch: 2.7.0nopt: 8.1.0@@ -24620,7 +24610,7 @@ snapshots:pathe: 2.0.3pkg-types: 2.3.0prompts: 2.4.2- semver: 7.7.4+ semver: 7.8.5'@nuxt/[email protected]([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))([email protected]([email protected]))':dependencies:@@ -24647,7 +24637,7 @@ snapshots:pathe: 2.0.3perfect-debounce: 2.0.0pkg-types: 2.3.0- semver: 7.7.4+ semver: 7.8.5simple-git: 3.30.0([email protected])sirv: 3.0.2structured-clone-es: 1.0.0@@ -24706,7 +24696,7 @@ snapshots:pkg-types: 2.3.0rc9: 2.1.2scule: 1.3.0- semver: 7.7.4+ semver: 7.8.5tinyglobby: 0.2.16ufo: 1.6.1unctx: 2.4.1@@ -25311,7 +25301,7 @@ snapshots:'@opentelemetry/instrumentation': 0.57.2(@opentelemetry/[email protected])([email protected])'@opentelemetry/semantic-conventions': 1.28.0forwarded-parse: 2.1.2- semver: 7.7.4+ semver: 7.8.5transitivePeerDependencies:- supports-color@@ -25531,7 +25521,7 @@ snapshots:'@types/shimmer': 1.2.0import-in-the-middle: 1.14.2require-in-the-middle: 7.4.0([email protected])- semver: 7.7.4+ semver: 7.8.5shimmer: 1.2.1transitivePeerDependencies:- supports-color@@ -28698,7 +28688,7 @@ snapshots:'@tailwindcss/[email protected]':dependencies:- detect-libc: 2.0.4+ detect-libc: 2.1.2tar: 7.4.3optionalDependencies:'@tailwindcss/oxide-android-arm64': 4.1.12@@ -32135,10 +32125,6 @@ snapshots:- [email protected]: {}-- [email protected]: {}-@@ -35161,7 +35147,7 @@ snapshots:dependencies:- detect-libc: 2.0.3+ detect-libc: 2.1.2optionalDependencies:lightningcss-darwin-arm64: 1.30.1lightningcss-darwin-x64: 1.30.1@@ -36269,7 +36255,7 @@ snapshots:pkg-types: 2.3.0postcss: 8.5.6postcss-nested: 7.0.2([email protected])- semver: 7.7.4+ semver: 7.8.5tinyglobby: 0.2.16optionalDependencies:typescript: 5.9.3@@ -36583,7 +36569,7 @@ snapshots:rollup: 4.52.5rollup-plugin-visualizer: 6.0.5([email protected](@emnapi/[email protected])(@emnapi/[email protected]))([email protected])scule: 1.3.0- semver: 7.7.4+ semver: 7.8.5serve-placeholder: 2.0.2serve-static: 2.2.0([email protected])source-map: 0.7.6@@ -36685,7 +36671,7 @@ snapshots:rollup: 4.52.5rollup-plugin-visualizer: 6.0.5([email protected](@emnapi/[email protected])(@emnapi/[email protected]))([email protected])scule: 1.3.0- semver: 7.7.4+ semver: 7.8.5serve-placeholder: 2.0.2serve-static: 2.2.0([email protected])source-map: 0.7.6@@ -39130,8 +39116,6 @@ snapshots:dependencies:lru-cache: 6.0.0- [email protected]: {}-@@ -39276,8 +39260,8 @@ snapshots:dependencies:color: 4.2.3- detect-libc: 2.0.3- semver: 7.6.3+ detect-libc: 2.1.2+ semver: 7.8.5optionalDependencies:'@img/sharp-darwin-arm64': 0.33.5'@img/sharp-darwin-x64': 0.33.5@@ -39303,7 +39287,7 @@ snapshots:dependencies:'@img/colour': 1.0.0detect-libc: 2.1.2- semver: 7.7.4+ semver: 7.8.5optionalDependencies:'@img/sharp-darwin-arm64': 0.34.5'@img/sharp-darwin-x64': 0.34.5