TagSpaces is an offline, open source, document manager with tagging support
feat: added simple content guessing for the downloaded url
Ilian Sapundshievaf3163ce04e9
diff --git a/src/main/mainEvents.ts b/src/main/mainEvents.tsindex feb3bc165..25d2f8560 100644--- a/src/main/mainEvents.ts+++ b/src/main/mainEvents.ts@@ -195,6 +195,34 @@ export default function loadMainEvents() {}});+ // Cheaply probe a URL's Content-Type (used to recognize PDFs/images served+ // from extension-less URLs, e.g. arxiv.org/pdf/...). Reads only the response+ // headers and cancels the body so the file isn't downloaded.+ ipcMain.handle('probeContentType', async (_event, url) => {+ if (!isHttpUrl(url)) {+ return { error: 'Unsupported URL scheme (only http/https allowed)' };+ }+ const controller = new AbortController();+ const timer = setTimeout(() => controller.abort(), 15000);+ try {+ const response = await net.fetch(url, {+ headers: browserHeaders(url),+ signal: controller.signal,+ });+ const contentType = response.headers.get('content-type') || '';+ try {+ await response.body?.cancel();+ } catch (e) {+ // ignore — we already have the header we need+ }+ return { contentType };+ } catch (error) {+ return { error: error.message };+ } finally {+ clearTimeout(timer);+ }+ });+// Fetch a URL and return its bytes (object-store locations + image inlining for// the HTML/Markdown "save as" formats). Same browser-stack request as fetchUrl.// `options` lets callers harden untrusted sub-resource fetches (page images):diff --git a/src/main/preload.ts b/src/main/preload.tsindex 9df64eb1e..48dabecae 100644--- a/src/main/preload.ts+++ b/src/main/preload.ts@@ -11,6 +11,7 @@ export type Channels =| 'isWorkerAvailable'| 'fetchUrl'| 'fetchUrlBuffer'+ | 'probeContentType'| 'isDirectory'| 'resolveRelativePaths'| 'set-language'diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsxindex d924975d1..6a3a4690a 100644--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx@@ -33,6 +33,7 @@ import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';import { useIOActionsContext } from '-/hooks/useIOActionsContext';import { useNotificationContext } from '-/hooks/useNotificationContext';+import { getExtensionForMimeType } from '-/services/utils-io';import { actions as AppActions, AppDispatch } from '-/reducers/app';import {getFileNameTagPlace,@@ -88,38 +89,43 @@ function sanitizeDownloadFileName(name: string): string {}// Split a sanitized filename into base name and extension (incl. leading dot).+// Only a real-looking extension counts (≤8 chars, contains a letter), so a+// numeric suffix such as the arXiv id "2605.22391" is kept as part of the name+// rather than mistaken for an extension.function splitNameExt(name: string): { base: string; ext: string } {const dot = name.lastIndexOf('.');- return dot > 0- ? { base: name.slice(0, dot), ext: name.slice(dot) }- : { base: name, ext: '' };+ if (dot > 0) {+ const ext = name.slice(dot);+ if (/^\.[a-z0-9]{1,8}$/i.test(ext) && /[a-z]/i.test(ext)) {+ return { base: name.slice(0, dot), ext };+ }+ }+ return { base: name, ext: '' };}-// Best-effort base name + original extension from a URL.+// Best-effort base name + extension from a URL. Uses the last non-empty path+// segment (so `…/blog/my-article` and `…/blog/my-article/` both yield+// `my-article`); for a bare host or "/" it falls back to the hostname. An+// extension-less segment (article slug) or the hostname defaults to `.html`,+// while a real file segment keeps its extension (`photo.jpg`, `report.pdf`).function deriveNameFromUrl(urlStr: string): { base: string; ext: string } {- let fileName: string;- let pathParts: string[];const url = new URL(urlStr);- if (url.pathname) {- const i = url.pathname.lastIndexOf('/');- if (i > -1) {- fileName = url.pathname.substring(i + 1);- if (!fileName) {- pathParts = url.pathname.split('/').filter(Boolean);- }- } else {- fileName = url.pathname;- }- }- if (!fileName) {- fileName = `${- url.hostname +- (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')- }.html`;- } else if (fileName.indexOf('.') === -1) {- fileName = `${url.hostname}-${fileName}.html`;+ const segments = url.pathname.split('/').filter(Boolean);++ if (segments.length > 0) {+ const last = sanitizeDownloadFileName(segments[segments.length - 1]);+ const { base, ext } = splitNameExt(last);+ return {+ base: base || sanitizeDownloadFileName(url.hostname) || 'download',+ ext: ext || '.html',+ };}- return splitNameExt(sanitizeDownloadFileName(fileName) || 'download.html');+ // No usable path — keep the whole hostname (e.g. "pi.dev") as the base name;+ // its dots aren't a file extension, so don't split on them.+ return {+ base: sanitizeDownloadFileName(url.hostname) || 'download',+ ext: '.html',+ };}// Extension implied by the chosen save format (clipped formats override the@@ -144,7 +150,7 @@ function DownloadUrlDialog(props: Props) {const smallScreen = useMediaQuery(theme.breakpoints.down('md'));const { setReflectActions } = useEditedEntryContext();const { currentLocation } = useCurrentLocationContext();- const { downloadUrl, downloadUrlAs, getUrlReaderable } =+ const { downloadUrl, downloadUrlAs, inspectUrl, probeContentType } =useIOActionsContext();const { showNotification } = useNotificationContext();const { openFileUploadDialog } = useFileUploadDialogContext();@@ -165,20 +171,26 @@ function DownloadUrlDialog(props: Props) {const [fileName, setFileName] = useState<string>('');const [tags, setTags] = useState<TS.Tag[]>([]);const inputTags = useRef<TS.Tag[]>([]);- // Original extension derived from the URL, used when format is "original".- const originalExt = useRef<string>('.html');+ // Source content's extension (from the URL or sniffed Content-Type), used as+ // the "original" extension and to decide whether conversion is allowed. State+ // so the convert gate re-renders when a PDF/image is recognized.+ const [originalExt, setOriginalExt] = useState<string>('.html');const [invalidURL, setInvalidURL] = useState<boolean>(false);const [saveFormat, setSaveFormat] = useState<SaveFormat>('original');const [extractArticle, setExtractArticle] = useState<boolean>(false);const [embedImages, setEmbedImages] = useState<boolean>(true);+ // HTML/Markdown conversion only makes sense for real HTML — disable it when+ // the source is a non-HTML file (PDF, image, …) by extension or Content-Type.+ const convertDisabled = !isHtmlExtension(originalExt);+// Reset the form and seed an automatic datetime-stamp tag each time the// (kept-mounted) dialog opens.useEffect(() => {if (open) {setUrl('');setFileName('');- originalExt.current = '.html';+ setOriginalExt('.html');setSaveFormat('original');setExtractArticle(false);setEmbedImages(true);@@ -197,25 +209,76 @@ function DownloadUrlDialog(props: Props) {// eslint-disable-next-line react-hooks/exhaustive-deps}, [open]);- // When a conversion format is selected, default the "Extract main article"- // switch to checked only if Readability considers the page a parseable- // article. Fetches the page (debounced) and caches the verdict per URL.- const readableCache = useRef<{ url: string; promise: Promise<boolean> }>();+ // On URL entry, cheaply probe the Content-Type (headers only) so a PDF/image+ // served from an extension-less URL (e.g. arxiv.org/pdf/...) is recognized:+ // the extension is corrected and conversion gets disabled. Debounced + cached.+ const probeCache = useRef<{ url: string; promise: Promise<string> }>();+ useEffect(() => {+ if (!open || !/^https?:\/\//i.test(url)) {+ return undefined;+ }+ let cancelled = false;+ const timer = setTimeout(() => {+ if (!probeCache.current || probeCache.current.url !== url) {+ probeCache.current = { url, promise: probeContentType(url) };+ }+ probeCache.current.promise+ .then((contentType) => {+ if (cancelled) {+ return undefined;+ }+ // Recognize binary types (PDF/image/zip/…) that can't be converted;+ // ignore HTML/unknown so an HTML page keeps its URL-derived name.+ const mapped = getExtensionForMimeType(contentType);+ const ext = mapped ? `.${mapped}` : null;+ if (ext && !isHtmlExtension(ext)) {+ setOriginalExt(ext);+ setSaveFormat('original');+ setFileName((prev) => splitNameExt(prev).base + ext);+ }+ return undefined;+ })+ .catch(() => {});+ }, 400);+ return () => {+ cancelled = true;+ clearTimeout(timer);+ };+ // probeContentType is a stable context fn.+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [url, open]);++ // When a conversion format is selected, fetch the page (debounced, cached per+ // URL) to (a) default "Extract main article" on only for a parseable article+ // and (b) detect a PDF body served without a Content-Type — revert to Original.+ const inspectCache = useRef<{+ url: string;+ promise: ReturnType<typeof inspectUrl>;+ }>();useEffect(() => {if (!open || saveFormat === 'original' || !/^https?:\/\//i.test(url)) {return undefined;}let cancelled = false;const timer = setTimeout(() => {- if (!readableCache.current || readableCache.current.url !== url) {- readableCache.current = { url, promise: getUrlReaderable(url) };+ if (!inspectCache.current || inspectCache.current.url !== url) {+ inspectCache.current = { url, promise: inspectUrl(url) };}- readableCache.current.promise- .then((readerable) => {- if (!cancelled) {- setExtractArticle(readerable);+ inspectCache.current.promise+ .then((info) => {+ if (cancelled) {+ return info;}- return readerable;+ if (info.isPdf) {+ // PDF body served from an HTML-looking URL — correct the extension+ // and fall back to a raw "Original" download.+ setOriginalExt('.pdf');+ setSaveFormat('original');+ setFileName((prev) => `${splitNameExt(prev).base}.pdf`);+ } else {+ setExtractArticle(info.readerable);+ }+ return info;}).catch(() => {});}, 500);@@ -223,7 +286,7 @@ function DownloadUrlDialog(props: Props) {cancelled = true;clearTimeout(timer);};- // getUrlReaderable is a stable context fn.+ // inspectUrl is a stable context fn.// eslint-disable-next-line react-hooks/exhaustive-deps}, [url, saveFormat, open]);@@ -239,7 +302,7 @@ function DownloadUrlDialog(props: Props) {}try {const { base, ext } = deriveNameFromUrl(value);- originalExt.current = ext;+ setOriginalExt(ext);setFileName(base + extForFormat(saveFormat, ext));} catch (e) {// incomplete/invalid URL — leave the filename as-is@@ -250,8 +313,7 @@ function DownloadUrlDialog(props: Props) {if (!value) return;setSaveFormat(value);setFileName(- (prev) =>- splitNameExt(prev).base + extForFormat(value, originalExt.current),+ (prev) => splitNameExt(prev).base + extForFormat(value, originalExt),);};@@ -305,8 +367,11 @@ function DownloadUrlDialog(props: Props) {// filename: name[tag1 tag2].extconst allTags = uniqueTags([...tags, ...inputTags.current]);const tagTitles = allTags.map((tag) => tag.title);+ // Re-sanitize the (user-editable) filename so a typed/pasted "../" can't+ // escape the target directory — generateFileName doesn't strip separators.const baseName =- fileName || `download${extForFormat(saveFormat, originalExt.current)}`;+ sanitizeDownloadFileName(fileName) ||+ `download${extForFormat(saveFormat, originalExt)}`;const finalName = generateFileName(baseName,tagTitles,@@ -488,11 +553,16 @@ function DownloadUrlDialog(props: Props) {>{t('core:originalFile')}</TsToggleButton>- <TsToggleButton value="html" data-tid="downloadFormatHtmlTID">+ <TsToggleButton+ value="html"+ disabled={convertDisabled}+ data-tid="downloadFormatHtmlTID"+ >{t('core:cleanedHtml')}</TsToggleButton><TsToggleButtonvalue="markdown"+ disabled={convertDisabled}data-tid="downloadFormatMarkdownTID">{t('core:markdown')}diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsxindex 7057d0618..91fe17993 100644--- a/src/renderer/hooks/IOActionsContextProvider.tsx+++ b/src/renderer/hooks/IOActionsContextProvider.tsx@@ -143,7 +143,8 @@ type IOActionsContextData = {stripStyles?: boolean;},) => Promise<TS.FileSystemEntry>;- getUrlReaderable: (url: string) => Promise<boolean>;+ inspectUrl: (url: string) => Promise<{ readerable: boolean; isPdf: boolean }>;+ probeContentType: (url: string) => Promise<string>;downloadFsEntry: (fsEntry: TS.FileSystemEntry) => void;uploadFilesAPI: (files: Array<any>,@@ -278,7 +279,8 @@ export const IOActionsContext = createContext<IOActionsContextData>({copyFiles: undefined,downloadUrl: undefined,downloadUrlAs: undefined,- getUrlReaderable: undefined,+ inspectUrl: undefined,+ probeContentType: undefined,downloadFsEntry: undefined,uploadFilesAPI: undefined,uploadMeta: undefined,@@ -1202,11 +1204,43 @@ export const IOActionsContextProvider = ({);}- /** Fetch a page and report whether Readability treats it as a parseable article. */- function getUrlReaderable(url: string): Promise<boolean> {+ /**+ * Cheaply probe a URL's Content-Type (headers only) to recognize PDFs/images+ * served from extension-less URLs (e.g. arxiv.org/pdf/...). Returns '' on+ * failure or where it isn't supported (web).+ */+ function probeContentType(url: string): Promise<string> {+ if (AppConfig.isElectron) {+ return window.electronIO.ipcRenderer+ .invoke('probeContentType', url)+ .then((res) => (res?.error ? '' : res?.contentType || ''))+ .catch(() => '');+ }+ if (AppConfig.isCapacitor) {+ // eslint-disable-next-line global-require+ const ioAPI = require('-/services/io-capacitor');+ return ioAPI+ .httpHead(url)+ .then((res: { contentType?: string }) => res?.contentType || '')+ .catch(() => '');+ }+ return Promise.resolve('');+ }++ /**+ * Fetch a page and report whether it's a parseable article (Readability) and+ * whether the body is actually a PDF (magic bytes) — used to gate the+ * HTML/Markdown conversion options, which only make sense for real HTML.+ */+ function inspectUrl(+ url: string,+ ): Promise<{ readerable: boolean; isPdf: boolean }> {return fetchPageHtml(url)- .then((html) => isHtmlReaderable(html))- .catch(() => false);+ .then((html) => ({+ isPdf: html.slice(0, 8).startsWith('%PDF-'),+ readerable: isHtmlReaderable(html),+ }))+ .catch(() => ({ readerable: false, isPdf: false }));}function downloadFsEntry(fsEntry: TS.FileSystemEntry) {@@ -2879,7 +2913,8 @@ export const IOActionsContextProvider = ({copyFiles,downloadUrl,downloadUrlAs,- getUrlReaderable,+ inspectUrl,+ probeContentType,downloadFsEntry,uploadFilesAPI,uploadFiles,diff --git a/src/renderer/services/io-capacitor.ts b/src/renderer/services/io-capacitor.tsindex 98a9d23ca..cae82e1ed 100644--- a/src/renderer/services/io-capacitor.ts+++ b/src/renderer/services/io-capacitor.ts@@ -1150,6 +1150,17 @@ function shareFiles(files) {});}+// Only allow http/https for native HTTP requests (defense-in-depth: reject+// file:/data:/other schemes before they reach the native plugin).+function isHttpUrl(url) {+ try {+ const { protocol } = new URL(url);+ return protocol === 'http:' || protocol === 'https:';+ } catch (e) {+ return false;+ }+}+/*** Fetch a remote URL over the native HTTP stack. WKWebView rejects cross-origin* fetch() from the capacitor:// origin with "Load failed"; CapacitorHttp runs@@ -1158,6 +1169,9 @@ function shareFiles(files) {* responseType), leaving byte/data-URL conversion to the caller.*/function httpGet(url) {+ if (!isHttpUrl(url)) {+ return Promise.reject(new Error('Unsupported URL scheme'));+ }return CapacitorHttp.request({url,method: 'GET',@@ -1176,6 +1190,22 @@ function httpGet(url) {});}+/**+ * Probe a URL's Content-Type via a native HEAD request (CORS-free). Used to+ * recognize PDFs/images served from extension-less URLs.+ */+function httpHead(url) {+ if (!isHttpUrl(url)) {+ return Promise.reject(new Error('Unsupported URL scheme'));+ }+ return CapacitorHttp.request({ url, method: 'HEAD' }).then((res) => {+ const headers = res.headers || {};+ return {+ contentType: headers['content-type'] || headers['Content-Type'] || '',+ };+ });+}+/*** Download a file under Capacitor. Replaces the Cordova-only* `window.plugins.Downloader`, which is undefined under Capacitor.@@ -1312,5 +1342,6 @@ export {shareFiles,downloadFile,httpGet,+ httpHead,getNativeFileUrlAsync,};diff --git a/src/renderer/services/utils-io.ts b/src/renderer/services/utils-io.tsindex 6e85051ea..0847971ce 100644--- a/src/renderer/services/utils-io.ts+++ b/src/renderer/services/utils-io.ts@@ -895,6 +895,14 @@ export function getDefaultEditor(fileType) {export function openUrl(url: string): void {if (AppConfig.isElectron) {window.electronIO.ipcRenderer.sendMessage('openUrl', url);+ } else if (AppConfig.isCapacitor) {+ // A Capacitor WebView can't handle target="_blank" / window.open+ // reliably — it either swallows the click or navigates the app itself+ // away. Delegate to the io-capacitor Browser.open path, which opens an+ // SFSafariViewController (iOS) / Custom Tab (Android) and returns the+ // user to the app afterwards.+ const ioAPI = require('-/services/io-capacitor');+ ioAPI.openUrl(url);} else {// web or cordovaopenUrlForWeb(url);@@ -1365,6 +1373,29 @@ export function getMimeType(extension: string): string | undefined {return MIME_TYPE_MAP[extension.toLowerCase()];}+// Reverse of MIME_TYPE_MAP — the canonical (first-listed) extension per MIME+// type, e.g. image/jpeg → "jpg", text/html → "html".+const EXTENSION_FOR_MIME_TYPE: Readonly<Record<string, string>> = Object.freeze(+ Object.entries(MIME_TYPE_MAP).reduce(+ (acc, [ext, mime]) => {+ if (!acc[mime]) acc[mime] = ext;+ return acc;+ },+ {} as Record<string, string>,+ ),+);++/**+ * Reverse of getMimeType: a canonical file extension (without leading dot) for a+ * Content-Type / MIME type, ignoring any "; charset=…" suffix. Returns undefined+ * for unknown types.+ */+export function getExtensionForMimeType(mimeType: string): string | undefined {+ if (!mimeType) return undefined;+ const type = mimeType.split(';')[0].trim().toLowerCase();+ return EXTENSION_FOR_MIME_TYPE[type];+}+export function toTsLocation(location: CommonLocation): TS.S3Location {// Build result with required fields firstconst result: any = {diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.jsindex a89609978..d328cf05d 100644--- a/tests/e2e/download-url.pw.e2e.js+++ b/tests/e2e/download-url.pw.e2e.js@@ -77,6 +77,10 @@ test.beforeAll(async ({ isWeb, isS3, webServerPort }, testInfo) => {} else if (req.url === '/page.html') {res.writeHead(200, { 'Content-Type': 'text/html' });res.end(PAGE_HTML);+ } else if (req.url === '/paper/2605.22391') {+ // PDF served from an extension-less URL (arxiv-style).+ res.writeHead(200, { 'Content-Type': 'application/pdf' });+ res.end('%PDF-1.4\nfake pdf body');} else {res.writeHead(404);res.end('not found');@@ -207,4 +211,23 @@ test.describe('TST54 - Download from URL', () => {expect(html).not.toContain('<style');expect(html).not.toContain('rebeccapurple');});++ test('TST5424 - PDF from an extension-less URL gets .pdf + convert disabled [electron]', async () => {+ await openDownloadUrlDialog();+ await setInputValue(+ '[data-tid=newUrlTID] input',+ baseUrl + '/paper/2605.22391',+ );+ // The Content-Type probe (debounced) recognizes application/pdf → the+ // filename gets a .pdf extension and conversion is disabled.+ await expect(+ global.client.locator('[data-tid=downloadFileNameTID] input'),+ ).toHaveValue(/\.pdf$/, { timeout: 10000 });+ await expect(+ global.client.locator('[data-tid=downloadFormatMarkdownTID]'),+ ).toBeDisabled();+ await expect(+ global.client.locator('[data-tid=downloadFormatHtmlTID]'),+ ).toBeDisabled();+ });});