OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
api: clean up favicon route
Carl-Gerhard Lindesvärd709449e55c17
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 348016a37..6bf076845 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -25,6 +25,7 @@"ramda": "^0.29.1","sharp": "^0.33.2","ua-parser-js": "^1.0.37",+ "url-metadata": "^4.1.0","uuid": "^9.0.1"},"devDependencies": {diff --git a/apps/api/src/controllers/misc.controller.ts b/apps/api/src/controllers/misc.controller.tsindex 2932b5bee..41b172ba3 100644--- a/apps/api/src/controllers/misc.controller.ts+++ b/apps/api/src/controllers/misc.controller.ts@@ -1,3 +1,5 @@+import { logger } from '@/utils/logger';+import { parseUrlMeta } from '@/utils/parseUrlMeta';import type { FastifyReply, FastifyRequest } from 'fastify';import icoToPng from 'ico-to-png';import sharp from 'sharp';@@ -35,8 +37,7 @@ async function getImageBuffer(url: string) {.png().toBuffer();} catch (e) {- console.log('Failed to get image from url', url);- console.log(e);+ logger.error(e, `Failed to get image from url ${url}`);}}@@ -53,8 +54,6 @@ export async function getFavicon(redis.set(`favicon:${cacheKey}`, buffer.toString('base64'));}reply.type('image/png');- console.log('buffer', buffer.byteLength);-return reply.send(buffer);}@@ -64,7 +63,6 @@ export async function getFavicon(const url = decodeURIComponent(request.query.url);- // DIRECT IMAGEif (imageExtensions.find((ext) => url.endsWith(ext))) {const cacheKey = createHash(url, 32);const cache = await redis.get(`favicon:${cacheKey}`);@@ -77,41 +75,28 @@ export async function getFavicon(}}- const { hostname, origin } = new URL(url);+ const { hostname } = new URL(url);const cache = await redis.get(`favicon:${hostname}`);+if (cache) {return sendBuffer(Buffer.from(cache, 'base64'));}- // TRY FAVICON.ICO- const buffer = await getImageBuffer(`${origin}/favicon.ico`);- if (buffer && buffer.byteLength > 0) {- return sendBuffer(buffer, hostname);- }-- // PARSE HTML- const res = await fetch(url).then((res) => res.text());-- function findFavicon(res: string) {- const match = res.match(- /(\<link(.+?)image\/x-icon(.+?)\>|\<link(.+?)shortcut\sicon(.+?)\>)/- );- if (!match) {- return null;- }-- return match[0].match(/href="(.+?)"/)?.[1] ?? null;- }-- const favicon = findFavicon(res);- if (favicon) {- const buffer = await getImageBuffer(favicon);-+ const meta = await parseUrlMeta(url);+ if (meta?.favicon) {+ const buffer = await getImageBuffer(meta.favicon);if (buffer && buffer.byteLength > 0) {return sendBuffer(buffer, hostname);}}+ const buffer = await getImageBuffer(+ 'https://www.iconsdb.com/icons/download/orange/warning-128.png'+ );+ if (buffer && buffer.byteLength > 0) {+ return sendBuffer(buffer, hostname);+ }+return reply.status(404).send('Not found');}diff --git a/apps/api/src/utils/parseUrlMeta.ts b/apps/api/src/utils/parseUrlMeta.tsnew file mode 100644index 000000000..a6c49735a--- /dev/null+++ b/apps/api/src/utils/parseUrlMeta.ts@@ -0,0 +1,38 @@+import urlMetadata from 'url-metadata';++function findBestFavicon(favicons: UrlMetaData['favicons']) {+ const match = favicons.find(+ (favicon) =>+ favicon.rel === 'shortcut icon' ||+ favicon.rel === 'icon' ||+ favicon.rel === 'apple-touch-icon'+ );+ if (match) {+ return match.href;+ }+ return null;+}++function transform(data: UrlMetaData, url: string) {+ const favicon = findBestFavicon(data.favicons);+ return {+ favicon: favicon ? new URL(favicon, url).toString() : null,+ };+}++interface UrlMetaData {+ favicons: {+ rel: string;+ href: string;+ sizes: string;+ }[];+}++export async function parseUrlMeta(url: string) {+ try {+ const metadata = (await urlMetadata(url)) as UrlMetaData;+ return transform(metadata, url);+ } catch (err) {+ return null;+ }+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 8bff57600..6c2fb50f9 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -68,6 +68,9 @@ importers:ua-parser-js:specifier: ^1.0.37version: 1.0.37+ url-metadata:+ specifier: ^4.1.0+ version: 4.1.0uuid:specifier: ^9.0.1version: 9.0.1@@ -7686,6 +7689,10 @@ packages:- supports-colordev: false+ resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}+ dev: false+resolution: {integrity: sha512-sXaHZicyEEmY86WyueLTQesbeoH/mquvarJaQNbjuOQO+7gbFcDEWqKmcWA4cOTLzFlfgvkiVxolk1k5bBIpmg==}dependencies:@@ -7974,6 +7981,30 @@ packages:resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}dev: false+ resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==}+ dependencies:+ boolbase: 1.0.0+ css-select: 5.1.0+ css-what: 6.1.0+ domelementtype: 2.3.0+ domhandler: 5.0.3+ domutils: 3.1.0+ dev: false++ resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}+ engines: {node: '>= 6'}+ dependencies:+ cheerio-select: 2.1.0+ dom-serializer: 2.0.0+ domhandler: 5.0.3+ domutils: 3.1.0+ htmlparser2: 8.0.2+ parse5: 7.1.2+ parse5-htmlparser2-tree-adapter: 7.0.0+ dev: false+resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}engines: {node: '>= 8.10.0'}@@ -8405,6 +8436,21 @@ packages:resolution: {integrity: sha512-COtn4EROW5dBGlE/4PiKnh6rZpAPxDeFLaEEwt4i10jpDMFt2EhQGS79QmmrO+iKCHv0PU/HrOWEhijFd1x99Q==}dev: false+ resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}+ dependencies:+ boolbase: 1.0.0+ css-what: 6.1.0+ domhandler: 5.0.3+ domutils: 3.1.0+ nth-check: 2.1.1+ dev: false++ resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}+ engines: {node: '>= 6'}+ dev: false+resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}engines: {node: '>=4'}@@ -8726,6 +8772,11 @@ packages:resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==}dev: false+ resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}+ engines: {node: '>= 12'}+ dev: false+resolution: {integrity: sha512-y8e109LYGgoQDveiEBD3DYXKba1jWf5BA8YU1FL5Tvm0BTdEfy54WLCwnuYWZNnzzvALy/QQ4Hov+Q9RVRv+Zw==}@@ -8981,10 +9032,37 @@ packages:csstype: 3.1.3dev: false+ resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}+ dependencies:+ domelementtype: 2.3.0+ domhandler: 5.0.3+ entities: 4.5.0+ dev: false++ resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==}+ dev: false++ resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==}+ engines: {node: '>= 4'}+ dependencies:+ domelementtype: 2.3.0+ dev: false+resolution: {integrity: sha512-uyb4NDIvQ3hRn6NiC+SIFaP4mJ/MdXlvtunaqK9Bn6dD3RuB/1S/gasEjDHD8eiaqdSael2vBv+hOs7Y+jhYOQ==}dev: false+ resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==}+ dependencies:+ dom-serializer: 2.0.0+ domelementtype: 2.3.0+ domhandler: 5.0.3+ dev: false+resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==}dependencies:@@ -10031,6 +10109,14 @@ packages:- encodingdev: false+ resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}+ engines: {node: ^12.20 || >= 14.13}+ dependencies:+ node-domexception: 1.0.0+ web-streams-polyfill: 3.3.3+ dev: false+resolution: {integrity: sha512-e6eB7zN6UBSwGVwrbWVH+gdLnkW9WwHhmq2YDK1Sh30pzx1onRVGBvogTlUeWxwTa+L86NYdo4hFkh7O8ZjSnA==}dev: false@@ -10192,6 +10278,13 @@ packages:engines: {node: '>=0.4.x'}dev: false+ resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}+ engines: {node: '>=12.20.0'}+ dependencies:+ fetch-blob: 3.2.0+ dev: false+resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}engines: {node: '>= 0.6'}@@ -10799,6 +10892,15 @@ packages:resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}dev: false+ resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}+ dependencies:+ domelementtype: 2.3.0+ domhandler: 5.0.3+ domutils: 3.1.0+ entities: 4.5.0+ dev: false+resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}engines: {node: '>= 0.8'}@@ -13390,6 +13492,11 @@ packages:minimatch: 3.1.2dev: false+ resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}+ engines: {node: '>=10.5.0'}+ dev: false+resolution: {integrity: sha512-VzW+TAk2wE4X9maiKMlT+GsPU4OMmR1U9CrHSmd3DFLn2IcZ9VJ6M6BBugGfYUnPCLSYxXdZy17M0BEJyhUTwg==}dev: false@@ -13406,6 +13513,15 @@ packages:whatwg-url: 5.0.0dev: false+ resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}+ dependencies:+ data-uri-to-buffer: 4.0.1+ fetch-blob: 3.2.0+ formdata-polyfill: 4.0.10+ dev: false+resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==}engines: {node: '>= 6.13.0'}@@ -13487,6 +13603,12 @@ packages:set-blocking: 2.0.0dev: false+ resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}+ dependencies:+ boolbase: 1.0.0+ dev: false+resolution: {integrity: sha512-2vPPEi+Z7WqML2jZYddDIfy5Dqb0r2fze2zTxNNknZaFpVHU3mFB3R+DWeJWGVx0ecvttSGlJTI+WG+8Z4cDWw==}dev: false@@ -13835,6 +13957,13 @@ packages:parse-path: 7.0.0dev: false+ resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==}+ dependencies:+ domhandler: 5.0.3+ parse5: 7.1.2+ dev: false+resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}dependencies:@@ -16632,6 +16761,14 @@ packages:resolution: {integrity: sha512-EGXjXJZhIHiQMK2pQukuFcL303nskqIRzWvPvV5O8miOfwoUb9G+a/Cld60kUyeaybEI94wvVClT10DtfeAExA==}dev: false+ resolution: {integrity: sha512-Ce6V/XzPUf5/0isAkn8EDLueV7+fDLnVPOf/42gudkKI4GF9q3ddMrROK61bTU+neB+LP33drToW+Fo+rJleMg==}+ engines: {node: '>=18.0.0'}+ dependencies:+ cheerio: 1.0.0-rc.12+ node-fetch: 3.3.2+ dev: false+resolution: {integrity: sha512-Lg4Vx1XZQauB42Hw3kK7JM6yjVjgFmFC5/Ab797s79aARomD2nEErc4mCgM8EZrARLmmbWpi5DGCadmK50DcAQ==}engines: {node: '>=10'}@@ -16837,6 +16974,11 @@ packages:resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==}dev: false+ resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}+ engines: {node: '>= 8'}+ dev: false+resolution: {integrity: sha512-BSR9wyRsy/KOValMgd5kMyr3JzpdeoR9KVId8u5GVlTTAtNChlsE4yTxeY7zMdNSyOmoKBv8NH2qeRY9Tg+IaA==}dev: false