Skip to content

TagSpaces is an offline, open source, document manager with tagging support

feat: added simple content guessing for the downloaded url

Ilian Sapundshievaf3163ce04e9
diff --git a/src/main/mainEvents.ts b/src/main/mainEvents.ts
index feb3bc165..25d2f8560 100644
--- a/src/main/mainEvents.ts
+++ b/src/main/mainEvents.ts
@@ -195,6 +195,34 @@ export default function loadMainEvents() {
}
});
+ // Cheaply probe a URL's Content-Type (used to recognize PDFs/images served
+ // from extension-less URLs, e.g. arxiv.org/pdf/...). Reads only the response
+ // headers and cancels the body so the file isn't downloaded.
+ ipcMain.handle('probeContentType', async (_event, url) => {
+ if (!isHttpUrl(url)) {
+ return { error: 'Unsupported URL scheme (only http/https allowed)' };
+ }
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), 15000);
+ try {
+ const response = await net.fetch(url, {
+ headers: browserHeaders(url),
+ signal: controller.signal,
+ });
+ const contentType = response.headers.get('content-type') || '';
+ try {
+ await response.body?.cancel();
+ } catch (e) {
+ // ignore — we already have the header we need
+ }
+ return { contentType };
+ } catch (error) {
+ return { error: error.message };
+ } finally {
+ clearTimeout(timer);
+ }
+ });
+
// Fetch a URL and return its bytes (object-store locations + image inlining for
// the HTML/Markdown "save as" formats). Same browser-stack request as fetchUrl.
// `options` lets callers harden untrusted sub-resource fetches (page images):
diff --git a/src/main/preload.ts b/src/main/preload.ts
index 9df64eb1e..48dabecae 100644
--- a/src/main/preload.ts
+++ b/src/main/preload.ts
@@ -11,6 +11,7 @@ export type Channels =
| 'isWorkerAvailable'
| 'fetchUrl'
| 'fetchUrlBuffer'
+ | 'probeContentType'
| 'isDirectory'
| 'resolveRelativePaths'
| 'set-language'
diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
index d924975d1..6a3a4690a 100644
--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx
+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
@@ -33,6 +33,7 @@ import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';
import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';
import { useIOActionsContext } from '-/hooks/useIOActionsContext';
import { useNotificationContext } from '-/hooks/useNotificationContext';
+import { getExtensionForMimeType } from '-/services/utils-io';
import { actions as AppActions, AppDispatch } from '-/reducers/app';
import {
getFileNameTagPlace,
@@ -88,38 +89,43 @@ function sanitizeDownloadFileName(name: string): string {
}
// Split a sanitized filename into base name and extension (incl. leading dot).
+// Only a real-looking extension counts (≤8 chars, contains a letter), so a
+// numeric suffix such as the arXiv id "2605.22391" is kept as part of the name
+// rather than mistaken for an extension.
function splitNameExt(name: string): { base: string; ext: string } {
const dot = name.lastIndexOf('.');
- return dot > 0
- ? { base: name.slice(0, dot), ext: name.slice(dot) }
- : { base: name, ext: '' };
+ if (dot > 0) {
+ const ext = name.slice(dot);
+ if (/^\.[a-z0-9]{1,8}$/i.test(ext) && /[a-z]/i.test(ext)) {
+ return { base: name.slice(0, dot), ext };
+ }
+ }
+ return { base: name, ext: '' };
}
-// Best-effort base name + original extension from a URL.
+// Best-effort base name + extension from a URL. Uses the last non-empty path
+// segment (so `…/blog/my-article` and `…/blog/my-article/` both yield
+// `my-article`); for a bare host or "/" it falls back to the hostname. An
+// extension-less segment (article slug) or the hostname defaults to `.html`,
+// while a real file segment keeps its extension (`photo.jpg`, `report.pdf`).
function deriveNameFromUrl(urlStr: string): { base: string; ext: string } {
- let fileName: string;
- let pathParts: string[];
const url = new URL(urlStr);
- if (url.pathname) {
- const i = url.pathname.lastIndexOf('/');
- if (i > -1) {
- fileName = url.pathname.substring(i + 1);
- if (!fileName) {
- pathParts = url.pathname.split('/').filter(Boolean);
- }
- } else {
- fileName = url.pathname;
- }
- }
- if (!fileName) {
- fileName = `${
- url.hostname +
- (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')
- }.html`;
- } else if (fileName.indexOf('.') === -1) {
- fileName = `${url.hostname}-${fileName}.html`;
+ const segments = url.pathname.split('/').filter(Boolean);
+
+ if (segments.length > 0) {
+ const last = sanitizeDownloadFileName(segments[segments.length - 1]);
+ const { base, ext } = splitNameExt(last);
+ return {
+ base: base || sanitizeDownloadFileName(url.hostname) || 'download',
+ ext: ext || '.html',
+ };
}
- return splitNameExt(sanitizeDownloadFileName(fileName) || 'download.html');
+ // No usable path — keep the whole hostname (e.g. "pi.dev") as the base name;
+ // its dots aren't a file extension, so don't split on them.
+ return {
+ base: sanitizeDownloadFileName(url.hostname) || 'download',
+ ext: '.html',
+ };
}
// Extension implied by the chosen save format (clipped formats override the
@@ -144,7 +150,7 @@ function DownloadUrlDialog(props: Props) {
const smallScreen = useMediaQuery(theme.breakpoints.down('md'));
const { setReflectActions } = useEditedEntryContext();
const { currentLocation } = useCurrentLocationContext();
- const { downloadUrl, downloadUrlAs, getUrlReaderable } =
+ const { downloadUrl, downloadUrlAs, inspectUrl, probeContentType } =
useIOActionsContext();
const { showNotification } = useNotificationContext();
const { openFileUploadDialog } = useFileUploadDialogContext();
@@ -165,20 +171,26 @@ function DownloadUrlDialog(props: Props) {
const [fileName, setFileName] = useState<string>('');
const [tags, setTags] = useState<TS.Tag[]>([]);
const inputTags = useRef<TS.Tag[]>([]);
- // Original extension derived from the URL, used when format is "original".
- const originalExt = useRef<string>('.html');
+ // Source content's extension (from the URL or sniffed Content-Type), used as
+ // the "original" extension and to decide whether conversion is allowed. State
+ // so the convert gate re-renders when a PDF/image is recognized.
+ const [originalExt, setOriginalExt] = useState<string>('.html');
const [invalidURL, setInvalidURL] = useState<boolean>(false);
const [saveFormat, setSaveFormat] = useState<SaveFormat>('original');
const [extractArticle, setExtractArticle] = useState<boolean>(false);
const [embedImages, setEmbedImages] = useState<boolean>(true);
+ // HTML/Markdown conversion only makes sense for real HTML — disable it when
+ // the source is a non-HTML file (PDF, image, …) by extension or Content-Type.
+ const convertDisabled = !isHtmlExtension(originalExt);
+
// Reset the form and seed an automatic datetime-stamp tag each time the
// (kept-mounted) dialog opens.
useEffect(() => {
if (open) {
setUrl('');
setFileName('');
- originalExt.current = '.html';
+ setOriginalExt('.html');
setSaveFormat('original');
setExtractArticle(false);
setEmbedImages(true);
@@ -197,25 +209,76 @@ function DownloadUrlDialog(props: Props) {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [open]);
- // When a conversion format is selected, default the "Extract main article"
- // switch to checked only if Readability considers the page a parseable
- // article. Fetches the page (debounced) and caches the verdict per URL.
- const readableCache = useRef<{ url: string; promise: Promise<boolean> }>();
+ // On URL entry, cheaply probe the Content-Type (headers only) so a PDF/image
+ // served from an extension-less URL (e.g. arxiv.org/pdf/...) is recognized:
+ // the extension is corrected and conversion gets disabled. Debounced + cached.
+ const probeCache = useRef<{ url: string; promise: Promise<string> }>();
+ useEffect(() => {
+ if (!open || !/^https?:\/\//i.test(url)) {
+ return undefined;
+ }
+ let cancelled = false;
+ const timer = setTimeout(() => {
+ if (!probeCache.current || probeCache.current.url !== url) {
+ probeCache.current = { url, promise: probeContentType(url) };
+ }
+ probeCache.current.promise
+ .then((contentType) => {
+ if (cancelled) {
+ return undefined;
+ }
+ // Recognize binary types (PDF/image/zip/…) that can't be converted;
+ // ignore HTML/unknown so an HTML page keeps its URL-derived name.
+ const mapped = getExtensionForMimeType(contentType);
+ const ext = mapped ? `.${mapped}` : null;
+ if (ext && !isHtmlExtension(ext)) {
+ setOriginalExt(ext);
+ setSaveFormat('original');
+ setFileName((prev) => splitNameExt(prev).base + ext);
+ }
+ return undefined;
+ })
+ .catch(() => {});
+ }, 400);
+ return () => {
+ cancelled = true;
+ clearTimeout(timer);
+ };
+ // probeContentType is a stable context fn.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [url, open]);
+
+ // When a conversion format is selected, fetch the page (debounced, cached per
+ // URL) to (a) default "Extract main article" on only for a parseable article
+ // and (b) detect a PDF body served without a Content-Type — revert to Original.
+ const inspectCache = useRef<{
+ url: string;
+ promise: ReturnType<typeof inspectUrl>;
+ }>();
useEffect(() => {
if (!open || saveFormat === 'original' || !/^https?:\/\//i.test(url)) {
return undefined;
}
let cancelled = false;
const timer = setTimeout(() => {
- if (!readableCache.current || readableCache.current.url !== url) {
- readableCache.current = { url, promise: getUrlReaderable(url) };
+ if (!inspectCache.current || inspectCache.current.url !== url) {
+ inspectCache.current = { url, promise: inspectUrl(url) };
}
- readableCache.current.promise
- .then((readerable) => {
- if (!cancelled) {
- setExtractArticle(readerable);
+ inspectCache.current.promise
+ .then((info) => {
+ if (cancelled) {
+ return info;
}
- return readerable;
+ if (info.isPdf) {
+ // PDF body served from an HTML-looking URL — correct the extension
+ // and fall back to a raw "Original" download.
+ setOriginalExt('.pdf');
+ setSaveFormat('original');
+ setFileName((prev) => `${splitNameExt(prev).base}.pdf`);
+ } else {
+ setExtractArticle(info.readerable);
+ }
+ return info;
})
.catch(() => {});
}, 500);
@@ -223,7 +286,7 @@ function DownloadUrlDialog(props: Props) {
cancelled = true;
clearTimeout(timer);
};
- // getUrlReaderable is a stable context fn.
+ // inspectUrl is a stable context fn.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [url, saveFormat, open]);
@@ -239,7 +302,7 @@ function DownloadUrlDialog(props: Props) {
}
try {
const { base, ext } = deriveNameFromUrl(value);
- originalExt.current = ext;
+ setOriginalExt(ext);
setFileName(base + extForFormat(saveFormat, ext));
} catch (e) {
// incomplete/invalid URL — leave the filename as-is
@@ -250,8 +313,7 @@ function DownloadUrlDialog(props: Props) {
if (!value) return;
setSaveFormat(value);
setFileName(
- (prev) =>
- splitNameExt(prev).base + extForFormat(value, originalExt.current),
+ (prev) => splitNameExt(prev).base + extForFormat(value, originalExt),
);
};
@@ -305,8 +367,11 @@ function DownloadUrlDialog(props: Props) {
// filename: name[tag1 tag2].ext
const allTags = uniqueTags([...tags, ...inputTags.current]);
const tagTitles = allTags.map((tag) => tag.title);
+ // Re-sanitize the (user-editable) filename so a typed/pasted "../" can't
+ // escape the target directory — generateFileName doesn't strip separators.
const baseName =
- fileName || `download${extForFormat(saveFormat, originalExt.current)}`;
+ sanitizeDownloadFileName(fileName) ||
+ `download${extForFormat(saveFormat, originalExt)}`;
const finalName = generateFileName(
baseName,
tagTitles,
@@ -488,11 +553,16 @@ function DownloadUrlDialog(props: Props) {
>
{t('core:originalFile')}
</TsToggleButton>
- <TsToggleButton value="html" data-tid="downloadFormatHtmlTID">
+ <TsToggleButton
+ value="html"
+ disabled={convertDisabled}
+ data-tid="downloadFormatHtmlTID"
+ >
{t('core:cleanedHtml')}
</TsToggleButton>
<TsToggleButton
value="markdown"
+ disabled={convertDisabled}
data-tid="downloadFormatMarkdownTID"
>
{t('core:markdown')}
diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsx
index 7057d0618..91fe17993 100644
--- a/src/renderer/hooks/IOActionsContextProvider.tsx
+++ b/src/renderer/hooks/IOActionsContextProvider.tsx
@@ -143,7 +143,8 @@ type IOActionsContextData = {
stripStyles?: boolean;
},
) => Promise<TS.FileSystemEntry>;
- getUrlReaderable: (url: string) => Promise<boolean>;
+ inspectUrl: (url: string) => Promise<{ readerable: boolean; isPdf: boolean }>;
+ probeContentType: (url: string) => Promise<string>;
downloadFsEntry: (fsEntry: TS.FileSystemEntry) => void;
uploadFilesAPI: (
files: Array<any>,
@@ -278,7 +279,8 @@ export const IOActionsContext = createContext<IOActionsContextData>({
copyFiles: undefined,
downloadUrl: undefined,
downloadUrlAs: undefined,
- getUrlReaderable: undefined,
+ inspectUrl: undefined,
+ probeContentType: undefined,
downloadFsEntry: undefined,
uploadFilesAPI: undefined,
uploadMeta: undefined,
@@ -1202,11 +1204,43 @@ export const IOActionsContextProvider = ({
);
}
- /** Fetch a page and report whether Readability treats it as a parseable article. */
- function getUrlReaderable(url: string): Promise<boolean> {
+ /**
+ * Cheaply probe a URL's Content-Type (headers only) to recognize PDFs/images
+ * served from extension-less URLs (e.g. arxiv.org/pdf/...). Returns '' on
+ * failure or where it isn't supported (web).
+ */
+ function probeContentType(url: string): Promise<string> {
+ if (AppConfig.isElectron) {
+ return window.electronIO.ipcRenderer
+ .invoke('probeContentType', url)
+ .then((res) => (res?.error ? '' : res?.contentType || ''))
+ .catch(() => '');
+ }
+ if (AppConfig.isCapacitor) {
+ // eslint-disable-next-line global-require
+ const ioAPI = require('-/services/io-capacitor');
+ return ioAPI
+ .httpHead(url)
+ .then((res: { contentType?: string }) => res?.contentType || '')
+ .catch(() => '');
+ }
+ return Promise.resolve('');
+ }
+
+ /**
+ * Fetch a page and report whether it's a parseable article (Readability) and
+ * whether the body is actually a PDF (magic bytes) — used to gate the
+ * HTML/Markdown conversion options, which only make sense for real HTML.
+ */
+ function inspectUrl(
+ url: string,
+ ): Promise<{ readerable: boolean; isPdf: boolean }> {
return fetchPageHtml(url)
- .then((html) => isHtmlReaderable(html))
- .catch(() => false);
+ .then((html) => ({
+ isPdf: html.slice(0, 8).startsWith('%PDF-'),
+ readerable: isHtmlReaderable(html),
+ }))
+ .catch(() => ({ readerable: false, isPdf: false }));
}
function downloadFsEntry(fsEntry: TS.FileSystemEntry) {
@@ -2879,7 +2913,8 @@ export const IOActionsContextProvider = ({
copyFiles,
downloadUrl,
downloadUrlAs,
- getUrlReaderable,
+ inspectUrl,
+ probeContentType,
downloadFsEntry,
uploadFilesAPI,
uploadFiles,
diff --git a/src/renderer/services/io-capacitor.ts b/src/renderer/services/io-capacitor.ts
index 98a9d23ca..cae82e1ed 100644
--- a/src/renderer/services/io-capacitor.ts
+++ b/src/renderer/services/io-capacitor.ts
@@ -1150,6 +1150,17 @@ function shareFiles(files) {
});
}
+// Only allow http/https for native HTTP requests (defense-in-depth: reject
+// file:/data:/other schemes before they reach the native plugin).
+function isHttpUrl(url) {
+ try {
+ const { protocol } = new URL(url);
+ return protocol === 'http:' || protocol === 'https:';
+ } catch (e) {
+ return false;
+ }
+}
+
/**
* Fetch a remote URL over the native HTTP stack. WKWebView rejects cross-origin
* fetch() from the capacitor:// origin with "Load failed"; CapacitorHttp runs
@@ -1158,6 +1169,9 @@ function shareFiles(files) {
* responseType), leaving byte/data-URL conversion to the caller.
*/
function httpGet(url) {
+ if (!isHttpUrl(url)) {
+ return Promise.reject(new Error('Unsupported URL scheme'));
+ }
return CapacitorHttp.request({
url,
method: 'GET',
@@ -1176,6 +1190,22 @@ function httpGet(url) {
});
}
+/**
+ * Probe a URL's Content-Type via a native HEAD request (CORS-free). Used to
+ * recognize PDFs/images served from extension-less URLs.
+ */
+function httpHead(url) {
+ if (!isHttpUrl(url)) {
+ return Promise.reject(new Error('Unsupported URL scheme'));
+ }
+ return CapacitorHttp.request({ url, method: 'HEAD' }).then((res) => {
+ const headers = res.headers || {};
+ return {
+ contentType: headers['content-type'] || headers['Content-Type'] || '',
+ };
+ });
+}
+
/**
* Download a file under Capacitor. Replaces the Cordova-only
* `window.plugins.Downloader`, which is undefined under Capacitor.
@@ -1312,5 +1342,6 @@ export {
shareFiles,
downloadFile,
httpGet,
+ httpHead,
getNativeFileUrlAsync,
};
diff --git a/src/renderer/services/utils-io.ts b/src/renderer/services/utils-io.ts
index 6e85051ea..0847971ce 100644
--- a/src/renderer/services/utils-io.ts
+++ b/src/renderer/services/utils-io.ts
@@ -895,6 +895,14 @@ export function getDefaultEditor(fileType) {
export function openUrl(url: string): void {
if (AppConfig.isElectron) {
window.electronIO.ipcRenderer.sendMessage('openUrl', url);
+ } else if (AppConfig.isCapacitor) {
+ // A Capacitor WebView can't handle target="_blank" / window.open
+ // reliably — it either swallows the click or navigates the app itself
+ // away. Delegate to the io-capacitor Browser.open path, which opens an
+ // SFSafariViewController (iOS) / Custom Tab (Android) and returns the
+ // user to the app afterwards.
+ const ioAPI = require('-/services/io-capacitor');
+ ioAPI.openUrl(url);
} else {
// web or cordova
openUrlForWeb(url);
@@ -1365,6 +1373,29 @@ export function getMimeType(extension: string): string | undefined {
return MIME_TYPE_MAP[extension.toLowerCase()];
}
+// Reverse of MIME_TYPE_MAP — the canonical (first-listed) extension per MIME
+// type, e.g. image/jpeg → "jpg", text/html → "html".
+const EXTENSION_FOR_MIME_TYPE: Readonly<Record<string, string>> = Object.freeze(
+ Object.entries(MIME_TYPE_MAP).reduce(
+ (acc, [ext, mime]) => {
+ if (!acc[mime]) acc[mime] = ext;
+ return acc;
+ },
+ {} as Record<string, string>,
+ ),
+);
+
+/**
+ * Reverse of getMimeType: a canonical file extension (without leading dot) for a
+ * Content-Type / MIME type, ignoring any "; charset=…" suffix. Returns undefined
+ * for unknown types.
+ */
+export function getExtensionForMimeType(mimeType: string): string | undefined {
+ if (!mimeType) return undefined;
+ const type = mimeType.split(';')[0].trim().toLowerCase();
+ return EXTENSION_FOR_MIME_TYPE[type];
+}
+
export function toTsLocation(location: CommonLocation): TS.S3Location {
// Build result with required fields first
const result: any = {
diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.js
index a89609978..d328cf05d 100644
--- a/tests/e2e/download-url.pw.e2e.js
+++ b/tests/e2e/download-url.pw.e2e.js
@@ -77,6 +77,10 @@ test.beforeAll(async ({ isWeb, isS3, webServerPort }, testInfo) => {
} else if (req.url === '/page.html') {
res.writeHead(200, { 'Content-Type': 'text/html' });
res.end(PAGE_HTML);
+ } else if (req.url === '/paper/2605.22391') {
+ // PDF served from an extension-less URL (arxiv-style).
+ res.writeHead(200, { 'Content-Type': 'application/pdf' });
+ res.end('%PDF-1.4\nfake pdf body');
} else {
res.writeHead(404);
res.end('not found');
@@ -207,4 +211,23 @@ test.describe('TST54 - Download from URL', () => {
expect(html).not.toContain('<style');
expect(html).not.toContain('rebeccapurple');
});
+
+ test('TST5424 - PDF from an extension-less URL gets .pdf + convert disabled [electron]', async () => {
+ await openDownloadUrlDialog();
+ await setInputValue(
+ '[data-tid=newUrlTID] input',
+ baseUrl + '/paper/2605.22391',
+ );
+ // The Content-Type probe (debounced) recognizes application/pdf → the
+ // filename gets a .pdf extension and conversion is disabled.
+ await expect(
+ global.client.locator('[data-tid=downloadFileNameTID] input'),
+ ).toHaveValue(/\.pdf$/, { timeout: 10000 });
+ await expect(
+ global.client.locator('[data-tid=downloadFormatMarkdownTID]'),
+ ).toBeDisabled();
+ await expect(
+ global.client.locator('[data-tid=downloadFormatHtmlTID]'),
+ ).toBeDisabled();
+ });
});