TagSpaces is an offline, open source, document manager with tagging support
feat: added url scrapping functionality
Ilian Sapundshievd4601037bce4
diff --git a/package-lock.json b/package-lock.jsonindex 5da02fb67..116f212a6 100644--- a/package-lock.json+++ b/package-lock.json@@ -26,6 +26,7 @@"@milkdown/crepe": "7.21.2","@milkdown/kit": "7.21.2","@milkdown/react": "7.21.2",+ "@mozilla/readability": "0.6.0","@mui/icons-material": "9.1.0","@mui/lab": "9.0.0-beta.4","@mui/material": "9.1.0",@@ -86,6 +87,7 @@"semver": "7.7.4","swiper": "12.1.3","tga-js": "1.1.1",+ "turndown": "7.2.4","utif.ts": "3.1.1","zod": "3.25.67","zod-to-json-schema": "3.25.1"@@ -103,6 +105,7 @@"@types/react-redux": "^7.1.24","@types/redux-logger": "^3.0.9","@types/terser-webpack-plugin": "^5.0.4",+ "@types/turndown": "^5.0.5","@types/webpack-bundle-analyzer": "^4.6.0","@typescript-eslint/eslint-plugin": "^6.7.0","@typescript-eslint/parser": "^6.7.0",@@ -6018,6 +6021,21 @@"nanoid": "^5.0.9"}},+ "node_modules/@mixmark-io/domino": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",+ "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",+ "license": "BSD-2-Clause"+ },+ "node_modules/@mozilla/readability": {+ "version": "0.6.0",+ "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",+ "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",+ "license": "Apache-2.0",+ "engines": {+ "node": ">=14.0.0"+ }+ },"node_modules/@mui/core-downloads-tracker": {"version": "9.1.0","resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-9.1.0.tgz",@@ -9906,6 +9924,13 @@"license": "MIT","optional": true},+ "node_modules/@types/turndown": {+ "version": "5.0.5",+ "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.5.tgz",+ "integrity": "sha512-TL2IgGgc7B5j78rIccBtlYAnkuv8nUQqhQc+DSYV5j9Be9XOcm/SKOVRuA47xAVI3680Tk9B1d8flK2GWT2+4w==",+ "dev": true,+ "license": "MIT"+ },"node_modules/@types/unist": {"version": "3.0.3","license": "MIT"@@ -29290,6 +29315,19 @@"node": "^20.17.0 || >=22.9.0"}},+ "node_modules/turndown": {+ "version": "7.2.4",+ "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",+ "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",+ "license": "MIT",+ "dependencies": {+ "@mixmark-io/domino": "^2.2.0"+ },+ "engines": {+ "node": ">=18",+ "npm": ">=9"+ }+ },"node_modules/type-check": {"version": "0.4.0","dev": true,diff --git a/package.json b/package.jsonindex 4dba975d5..bfccb87c9 100644--- a/package.json+++ b/package.json@@ -184,6 +184,7 @@"@milkdown/crepe": "7.21.2","@milkdown/kit": "7.21.2","@milkdown/react": "7.21.2",+ "@mozilla/readability": "0.6.0","@mui/icons-material": "9.1.0","@mui/lab": "9.0.0-beta.4","@mui/material": "9.1.0",@@ -244,6 +245,7 @@"semver": "7.7.4","swiper": "12.1.3","tga-js": "1.1.1",+ "turndown": "7.2.4","utif.ts": "3.1.1","zod": "3.25.67","zod-to-json-schema": "3.25.1"@@ -264,6 +266,7 @@"@types/react-redux": "^7.1.24","@types/redux-logger": "^3.0.9","@types/terser-webpack-plugin": "^5.0.4",+ "@types/turndown": "^5.0.5","@types/webpack-bundle-analyzer": "^4.6.0","@typescript-eslint/eslint-plugin": "^6.7.0","@typescript-eslint/parser": "^6.7.0",diff --git a/src/main/mainEvents.ts b/src/main/mainEvents.tsindex 24145e88c..feb3bc165 100644--- a/src/main/mainEvents.ts+++ b/src/main/mainEvents.ts@@ -30,9 +30,8 @@ import {isDirectory,} from '@tagspaces/tagspaces-common-node/io-node';import fs from 'fs-extra';-import http from 'http';-import https from 'https';import path from 'path';+import { Readable } from 'stream';import WebSocket from 'ws';import {getOnProgress,@@ -49,6 +48,37 @@ import registerSecureStorageEvents from './secureStorage';//let watcher: FSWatcher;const progress = {};++// A desktop-Chrome User-Agent so remote servers treat our downloads like a+// regular browser request. Many sites reject the default Electron/Node agent+// (403 "non-browser host"), which is why direct downloads previously failed.+const BROWSER_USER_AGENT =+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';++function browserHeaders(url: string): Record<string, string> {+ const headers: Record<string, string> = {+ 'User-Agent': BROWSER_USER_AGENT,+ Accept: '*/*',+ };+ try {+ headers.Referer = new URL(url).origin;+ } catch (e) {+ // invalid URL — net.fetch will reject below+ }+ return headers;+}++// Only allow http/https downloads. Without this, net.fetch would also resolve+// file:// (local file read) and other schemes — an SSRF / local-file-read risk,+// especially since image URLs are taken from fetched (untrusted) page content.+function isHttpUrl(url: string): boolean {+ try {+ const { protocol } = new URL(url);+ return protocol === 'http:' || protocol === 'https:';+ } catch (e) {+ return false;+ }+}let wsc;const controllers = new Map(); // requestId -> AbortController@@ -112,59 +142,104 @@ export default function loadMainEvents() {return `data:${contentType};base64,${base64}`;});- ipcMain.handle('fetchUrl', async (event, url, targetPath, withProgress) => {- function fetchHttp(url) {- return new Promise((resolve, reject) => {- const client = url.startsWith('https') ? https : http;-- client- .get(url, (res) => {- let downloadedSize = 0;- const totalSize = parseInt(res.headers['content-length'], 10);-- let onUploadProgress = undefined;- if (withProgress) {- progress['fetchUrl'] = newProgress('fetchUrl', totalSize);- onUploadProgress = getOnProgress('fetchUrl', progress);- }-- const fileStream = fs.createWriteStream(targetPath); // Write file to disk- // Pipe the response data into the file- res.pipe(fileStream);+ // Download a URL straight to disk (local locations). Uses Electron's net.fetch+ // (Chromium network stack — shares session/cookies, follows redirects, honors+ // proxy/gzip, and is not bound by the renderer's CSP or the server's CORS) with+ // browser-like headers so servers don't block us as a non-browser host.+ ipcMain.handle('fetchUrl', async (_event, url, targetPath, withProgress) => {+ if (!isHttpUrl(url)) {+ return { error: 'Unsupported URL scheme (only http/https allowed)' };+ }+ try {+ const response = await net.fetch(url, { headers: browserHeaders(url) });+ if (!response.ok) {+ return { error: `${response.status} ${response.statusText}` };+ }+ const contentType = response.headers.get('content-type') || '';+ const totalSize = parseInt(+ response.headers.get('content-length') || '',+ 10,+ );- // Collect data chunks and calculate size- res.on('data', (chunk) => {- downloadedSize += chunk.length; // Add the size of the current chunk- if (onUploadProgress) {- onUploadProgress(- { key: targetPath, loaded: downloadedSize, total: totalSize },- undefined,- );- }- });+ let onUploadProgress;+ if (withProgress) {+ progress['fetchUrl'] = newProgress('fetchUrl', totalSize);+ onUploadProgress = getOnProgress('fetchUrl', progress);+ }- // Resolve the response on end- res.on('end', () => {- if (onUploadProgress) {- onUploadProgress(- { key: targetPath, loaded: 1, total: 1 },- undefined,- );- }- resolve({ success: true, filePath: targetPath });- });- })- .on('error', (err) => {- reject(err);- });+ await new Promise<void>((resolve, reject) => {+ const fileStream = fs.createWriteStream(targetPath);+ const nodeStream = Readable.fromWeb(response.body as any);+ let downloadedSize = 0;+ nodeStream.on('data', (chunk) => {+ downloadedSize += chunk.length;+ if (onUploadProgress) {+ onUploadProgress(+ { key: targetPath, loaded: downloadedSize, total: totalSize },+ undefined,+ );+ }+ });+ nodeStream.on('error', reject);+ fileStream.on('error', reject);+ fileStream.on('finish', resolve);+ nodeStream.pipe(fileStream);});++ if (onUploadProgress) {+ onUploadProgress({ key: targetPath, loaded: 1, total: 1 }, undefined);+ }+ return { success: true, filePath: targetPath, contentType };+ } catch (error) {+ return { error: error.message };}+ });+ // Fetch a URL and return its bytes (object-store locations + image inlining for+ // the HTML/Markdown "save as" formats). Same browser-stack request as fetchUrl.+ // `options` lets callers harden untrusted sub-resource fetches (page images):+ // omitCredentials — don't send the app session's cookies to the target+ // maxBytes — reject oversized responses (memory/disk DoS guard)+ // timeoutMs — abort hung requests+ ipcMain.handle('fetchUrlBuffer', async (_event, url, options = {}) => {+ if (!isHttpUrl(url)) {+ return { error: 'Unsupported URL scheme (only http/https allowed)' };+ }+ const { omitCredentials, maxBytes, timeoutMs } = options;+ const controller = new AbortController();+ const timer = timeoutMs+ ? setTimeout(() => controller.abort(), timeoutMs)+ : undefined;try {- const response = await fetchHttp(url);- return response;+ const response = await net.fetch(url, {+ headers: browserHeaders(url),+ credentials: omitCredentials ? 'omit' : undefined,+ signal: controller.signal,+ });+ if (!response.ok) {+ return { error: `${response.status} ${response.statusText}` };+ }+ if (maxBytes) {+ const declared = parseInt(+ response.headers.get('content-length') || '',+ 10,+ );+ if (Number.isFinite(declared) && declared > maxBytes) {+ return { error: 'Response too large' };+ }+ }+ const contentType = response.headers.get('content-type') || '';+ const data = await response.arrayBuffer();+ if (maxBytes && data.byteLength > maxBytes) {+ return { error: 'Response too large' };+ }+ return { data, contentType };} catch (error) {return { error: error.message };+ } finally {+ if (timer) {+ clearTimeout(timer);+ }}});ipcMain.handle('isWorkerAvailable', async () => {diff --git a/src/main/preload.ts b/src/main/preload.tsindex 14973bcd6..9df64eb1e 100644--- a/src/main/preload.ts+++ b/src/main/preload.ts@@ -10,6 +10,7 @@ import {export type Channels =| 'isWorkerAvailable'| 'fetchUrl'+ | 'fetchUrlBuffer'| 'isDirectory'| 'resolveRelativePaths'| 'set-language'diff --git a/src/renderer/components/MobileNavigation.tsx b/src/renderer/components/MobileNavigation.tsxindex 9db98280b..88445302d 100644--- a/src/renderer/components/MobileNavigation.tsx+++ b/src/renderer/components/MobileNavigation.tsx@@ -42,7 +42,7 @@ import {} from '-/components/CommonIcons';import CustomLogo from '-/components/CustomLogo';import HelpFeedbackPanel from '-/components/HelpFeedbackPanel';-import { ProLabel } from '-/components/HelperComponents';+import { BetaLabel, ProLabel } from '-/components/HelperComponents';import InfoIcon from '-/components/InfoIcon';import LocationManager from '-/components/LocationManager';import ProTeaser from '-/components/ProTeaser';@@ -511,25 +511,27 @@ function MobileNavigation(props: Props) {</ListItemIcon><ListItemText primary={t('core:addFiles')} /></MenuItem>- {AppConfig.isElectron &&- !currentLocation?.haveObjectStoreSupport() && (- <MenuItem- key="newFromDownloadURL"- data-tid="newFromDownloadURLTID"- onClick={() => {- openDownloadUrl();- setOpenCreateMenu(false);- hideDrawer?.();- }}- >- <ListItemIcon>- <DownloadIcon />- </ListItemIcon>- <ListItemText- primary={t('core:newFromDownloadURL')}- />- </MenuItem>- )}+ <MenuItem+ key="newFromDownloadURL"+ data-tid="newFromDownloadURLTID"+ onClick={() => {+ openDownloadUrl();+ setOpenCreateMenu(false);+ hideDrawer?.();+ }}+ >+ <ListItemIcon>+ <DownloadIcon />+ </ListItemIcon>+ <ListItemText+ primary={+ <>+ {t('core:newFromDownloadURL')}+ <BetaLabel />+ </>+ }+ />+ </MenuItem><Divider /><MenuItemkey="createNewFolder"diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsxindex 73c7c6107..aeb756df2 100644--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx@@ -20,18 +20,25 @@ import AppConfig from '-/AppConfig';import DraggablePaper from '-/components/DraggablePaper';import TsButton from '-/components/TsButton';import TsTextField from '-/components/TsTextField';+import TsToggleButton from '-/components/TsToggleButton';import TargetPath from '-/components/dialogs/components/TargetPath';import TsDialogActions from '-/components/dialogs/components/TsDialogActions';import TsDialogTitle from '-/components/dialogs/components/TsDialogTitle';import { TargetPathContextProvider } from '-/components/dialogs/hooks/TargetPathContextProvider';import { useFileUploadDialogContext } from '-/components/dialogs/hooks/useFileUploadDialogContext';import { useTargetPathContext } from '-/components/dialogs/hooks/useTargetPathContext';-import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';import { useIOActionsContext } from '-/hooks/useIOActionsContext';import { useNotificationContext } from '-/hooks/useNotificationContext';import { actions as AppActions, AppDispatch } from '-/reducers/app';import { TS } from '-/tagspaces.namespace';+import {+ Box,+ Checkbox,+ FormControlLabel,+ ToggleButtonGroup,+ Typography,+} from '@mui/material';import Dialog from '@mui/material/Dialog';import DialogContent from '@mui/material/DialogContent';import Paper from '@mui/material/Paper';@@ -47,22 +54,47 @@ interface Props {onClose: (event?: Object, reason?: string) => void;}+// Derive a safe, single-segment filename from a URL-derived name. Decoding the+// raw name can reintroduce path separators / `..` (e.g. "%2e%2e%2f"), so keep+// only the final segment and drop traversal/control characters to prevent+// writing outside the chosen target directory.+function sanitizeDownloadFileName(name: string): string {+ let decoded = name;+ try {+ decoded = decodeURIComponent(name);+ } catch (e) {+ // keep the raw name if it isn't valid percent-encoding+ }+ const base = decoded.split(/[/\\]/).pop() || '';+ const cleaned = Array.from(base)+ .filter((ch) => ch.charCodeAt(0) >= 32)+ .join('');+ return /^\.+$/.test(cleaned) ? '' : cleaned;+}+function DownloadUrlDialog(props: Props) {const { open, onClose } = props;const { t } = useTranslation();const theme = useTheme();const smallScreen = useMediaQuery(theme.breakpoints.down('md'));- const { currentLocationId, findLocation, getFirstRWLocation } =- useCurrentLocationContext();const { setReflectActions } = useEditedEntryContext();- const { downloadUrl } = useIOActionsContext();+ const { downloadUrl, downloadUrlAs } = useIOActionsContext();const { showNotification } = useNotificationContext();const { openFileUploadDialog } = useFileUploadDialogContext();const dispatch: AppDispatch = useDispatch();+ // Cleaned-HTML / Markdown conversion needs CORS/CSP-free fetching of the page+ // and its images — only the desktop and native-mobile builds can do that.+ const canConvert = AppConfig.isElectron || AppConfig.isNativeMobile;+const { targetDirectoryPath } = useTargetPathContext();const fileUrl = useRef<string>();const [invalidURL, setInvalidURL] = useState<boolean>(false);+ const [saveFormat, setSaveFormat] = useState<+ 'original' | 'html' | 'markdown'+ >('original');+ const [extractArticle, setExtractArticle] = useState<boolean>(false);+ const [embedImages, setEmbedImages] = useState<boolean>(true);const onUploadProgress = (progress, abort, fileName) => {dispatch(AppActions.onUploadProgress(progress, abort, fileName));@@ -74,9 +106,6 @@ function DownloadUrlDialog(props: Props) {function downloadURL() {if (fileUrl.current) {- const location = currentLocationId- ? findLocation()- : getFirstRWLocation();try {const url = new URL(fileUrl.current);if (invalidURL) {@@ -96,51 +125,67 @@ function DownloadUrlDialog(props: Props) {}}if (!fileName) {- fileName =+ fileName = `${url.hostname +- (pathParts && pathParts.length > 0 ? pathParts.join('-') : '') +- '.html';+ (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')+ }.html`;} else if (fileName.indexOf('.') === -1) {- fileName = url.hostname + '-' + fileName + '.html';+ fileName = `${url.hostname}-${fileName}.html`;}- if (!location.haveObjectStoreSupport() && AppConfig.isElectron) {++ const reflectOpen = (entry: TS.FileSystemEntry) => {+ const reflectAction: TS.EditAction = {+ action: 'add',+ entry,+ open: true,+ source: 'upload',+ };+ setReflectActions(reflectAction);+ };+ const notifyError = (e: Error) => {+ console.log('downloadFile error:', e);+ dispatch(+ AppActions.setProgress(fileUrl.current, -1, t('core:errorCORS')),+ );+ showNotification(+ t('core:downloadFileError', { message: e.message }),+ 'error',+ true,+ );+ };++ const safeFileName =+ sanitizeDownloadFileName(fileName) || 'download.html';++ if (canConvert && saveFormat !== 'original') {+ // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline+ // images and optionally extract the article, then save into the location.+ const baseName = safeFileName.replace(/\.[^.]+$/, '') || 'download';+ const ext = saveFormat === 'markdown' ? '.md' : '.html';+ downloadUrlAs(+ fileUrl.current,+ `${targetDirectoryPath}/${baseName}${ext}`,+ saveFormat,+ { extractArticle, embedImages },+ )+ .then(reflectOpen)+ .catch(notifyError);+ } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {+ // Raw download straight into the location (local + cloud + mobile),+ // routed through the platform's CORS/CSP-free fetch.dispatch(AppActions.resetProgress());openFileUploadDialog();downloadUrl(fileUrl.current,- targetDirectoryPath + '/' + decodeURIComponent(fileName),+ `${targetDirectoryPath}/${safeFileName}`,onUploadProgress,)- .then((entry) => {- /*if (location?.haveObjectStoreSupport()) {- // currently objectStore location in downloadFile use saveFilePromise and this function not have progress handling- dispatch(AppActions.setProgress(fileUrl.current, 100));- }*/- const reflectAction: TS.EditAction = {- action: 'add',- entry: entry,- open: true,- source: 'upload',- };- setReflectActions(reflectAction);- })- .catch((e) => {- console.log('downloadFile error:', e);- dispatch(- AppActions.setProgress(- fileUrl.current,- -1,- t('core:errorCORS'),- ),- );- showNotification(- t('core:downloadFileError', { message: e.message }),- 'error',- true,- );- });+ .then(reflectOpen)+ .catch(notifyError);} else {- saveAs(fileUrl.current, decodeURIComponent(fileName));+ // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain+ // browser download.+ saveAs(fileUrl.current, safeFileName);}onClose();} catch (ex) {@@ -153,7 +198,7 @@ function DownloadUrlDialog(props: Props) {const okButton = (<TsButtonvariant="contained"- data-tid={'downloadFileUrlTID'}+ data-tid="downloadFileUrlTID"onClick={() => downloadURL()}sx={{@@ -204,6 +249,68 @@ function DownloadUrlDialog(props: Props) {onChange={handleUrlChange}/><TargetPath />+ {canConvert && (+ <Box sx={{ mt: 2 }}>+ <Typography+ variant="caption"+ color="textSecondary"+ sx={{ display: 'block', mb: 0.5 }}+ >+ {t('core:saveAs')}+ </Typography>+ <ToggleButtonGroup+ size="small"+ value={saveFormat}+ exclusive+ onChange={(_, v) => {+ if (v) setSaveFormat(v);+ }}+ data-tid="downloadFormatToggleTID"+ >+ <TsToggleButton+ value="original"+ data-tid="downloadFormatOriginalTID"+ >+ {t('core:originalFile')}+ </TsToggleButton>+ <TsToggleButton value="html" data-tid="downloadFormatHtmlTID">+ {t('core:cleanedHtml')}+ </TsToggleButton>+ <TsToggleButton+ value="markdown"+ data-tid="downloadFormatMarkdownTID"+ >+ {t('core:markdown')}+ </TsToggleButton>+ </ToggleButtonGroup>+ {saveFormat !== 'original' && (+ <Box sx={{ mt: 1 }}>+ <FormControlLabel+ control={+ <Checkbox+ size="small"+ checked={extractArticle}+ onChange={(e) => setExtractArticle(e.target.checked)}+ data-tid="downloadExtractArticleTID"+ />+ }+ label={t('core:extractMainArticle')}+ />+ <FormControlLabel+ control={+ <Checkbox+ size="small"+ checked={embedImages}+ onChange={(e) => setEmbedImages(e.target.checked)}+ data-tid="downloadEmbedImagesTID"+ />+ }+ label={t('core:embedImages')}+ />+ </Box>+ )}+ </Box>+ )}</DialogContent>{!smallScreen && (<TsDialogActions>diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsxindex fbf7c4e84..0575678a9 100644--- a/src/renderer/hooks/IOActionsContextProvider.tsx+++ b/src/renderer/hooks/IOActionsContextProvider.tsx@@ -48,6 +48,11 @@ import {openFileMessage,runConcurrent,} from '-/services/utils-io';+import {+ ClipOptions,+ htmlToCleanHtml,+ htmlToMarkdown,+} from '-/services/web-content';import { TS } from '-/tagspaces.namespace';import { CommonLocation } from '-/utils/CommonLocation';import { base64ToUint8Array } from '-/utils/dom';@@ -126,6 +131,16 @@ type IOActionsContextData = {targetPath: string,onDownloadProgress?: (progress, abort, fileName?) => void,) => Promise<TS.FileSystemEntry>;+ downloadUrlAs: (+ url: string,+ targetPath: string,+ format: 'html' | 'markdown',+ options?: {+ extractArticle?: boolean;+ embedImages?: boolean;+ tags?: string;+ },+ ) => Promise<TS.FileSystemEntry>;downloadFsEntry: (fsEntry: TS.FileSystemEntry) => void;uploadFilesAPI: (files: Array<any>,@@ -259,6 +274,7 @@ export const IOActionsContext = createContext<IOActionsContextData>({copyDirs: undefined,copyFiles: undefined,downloadUrl: undefined,+ downloadUrlAs: undefined,downloadFsEntry: undefined,uploadFilesAPI: undefined,uploadMeta: undefined,@@ -946,17 +962,61 @@ export const IOActionsContextProvider = ({.catch(() => {});}- function fetchUrl(url: string, targetPath: string, haveProgress: boolean) {+ /**+ * Native HTTP GET on Capacitor (bypasses the WKWebView CORS that makes+ * cross-origin fetch() fail with "Load failed"). The native call lives in the+ * Capacitor IO module; this is just the platform indirection.+ */+ function capacitorHttpGet(+ url: string,+ ): Promise<{ base64: string; contentType: string }> {+ // eslint-disable-next-line global-require+ const ioAPI = require('-/services/io-capacitor');+ return ioAPI.httpGet(url);+ }++ function fetchUrl(+ url: string,+ targetPath: string,+ haveProgress: boolean,+ objectStore: boolean,+ ) {if (AppConfig.isElectron) {- return window.electronIO.ipcRenderer.invoke(- 'fetchUrl',- url,- targetPath,- haveProgress,- );+ // Object-store targets need the bytes back in the renderer so they can be+ // uploaded into the bucket; local targets are streamed straight to disk in+ // the main process. Both requests go through the Chromium network stack+ // (browser User-Agent, no renderer CSP / server CORS limits).+ if (objectStore) {+ return window.electronIO.ipcRenderer+ .invoke('fetchUrlBuffer', url)+ .then((res) => {+ if (res?.error) {+ throw new Error(res.error);+ }+ return { arrayBuffer: () => Promise.resolve(res.data) };+ });+ }+ return window.electronIO.ipcRenderer+ .invoke('fetchUrl', url, targetPath, haveProgress)+ .then((res) => {+ if (res?.error) {+ throw new Error(res.error);+ }+ return res;+ });+ }+ if (AppConfig.isCapacitor) {+ // WKWebView rejects cross-origin fetch() from the capacitor:// origin+ // ("Load failed"). Route through the native HTTP plugin, which isn't bound+ // by CORS, and expose an arrayBuffer() so saveFile() takes the same path+ // as the web/object-store branch.+ return capacitorHttpGet(url).then(({ base64 }) => ({+ arrayBuffer: () => Promise.resolve(base64ToUint8Array(base64)),+ }));}return fetch(url);}+/*** @param url* @param targetPath@@ -985,7 +1045,12 @@ export const IOActionsContextProvider = ({return saveFilePromise({ path: targetPath }, arrayBuffer, true);});}- return fetchUrl(url, targetPath, onDownloadProgress !== undefined)+ return fetchUrl(+ url,+ targetPath,+ onDownloadProgress !== undefined,+ location.haveObjectStoreSupport(),+ ).then((response) => saveFile(response)).then((fsEntry: TS.FileSystemEntry) => {return generateThumbnailPromise(@@ -1020,6 +1085,111 @@ export const IOActionsContextProvider = ({});}+ /** Fetch a page's HTML, bypassing CORS/CSP via the main process on Electron. */+ function fetchPageHtml(url: string): Promise<string> {+ if (AppConfig.isElectron) {+ return window.electronIO.ipcRenderer+ .invoke('fetchUrlBuffer', url)+ .then((res) => {+ if (res?.error) {+ throw new Error(res.error);+ }+ return new TextDecoder('utf-8').decode(res.data);+ });+ }+ if (AppConfig.isCapacitor) {+ return capacitorHttpGet(url).then(({ base64 }) =>+ new TextDecoder('utf-8').decode(base64ToUint8Array(base64)),+ );+ }+ return fetch(url).then((r) => {+ if (!r.ok) {+ throw new Error(`${r.status} ${r.statusText}`);+ }+ return r.text();+ });+ }++ function blobToDataUrl(blob: Blob): Promise<string> {+ return new Promise((resolve) => {+ const reader = new FileReader();+ reader.onloadend = () => resolve(reader.result as string);+ reader.readAsDataURL(blob);+ });+ }++ /**+ * Resolve an image URL to a base64 data URL (CORS/CSP-free on Electron).+ * Image URLs come from untrusted page content, so fetch them without the app+ * session's cookies and with size/time caps to limit SSRF/DoS exposure.+ */+ function fetchImageDataUrl(url: string): Promise<string | null> {+ if (AppConfig.isElectron) {+ return window.electronIO.ipcRenderer+ .invoke('fetchUrlBuffer', url, {+ omitCredentials: true,+ maxBytes: 15 * 1024 * 1024,+ timeoutMs: 30000,+ })+ .then((res) => {+ if (res?.error || !res?.data) {+ return null;+ }+ const blob = new Blob([res.data], {+ type: res.contentType || 'application/octet-stream',+ });+ return blobToDataUrl(blob);+ })+ .catch(() => null);+ }+ if (AppConfig.isCapacitor) {+ // Build the data URL straight from the native base64 body.+ return capacitorHttpGet(url)+ .then(({ base64, contentType }) =>+ base64 ? `data:${contentType || 'image/*'};base64,${base64}` : null,+ )+ .catch(() => null);+ }+ return fetch(url, { credentials: 'omit' })+ .then((r) => r.blob())+ .then(blobToDataUrl)+ .catch(() => null);+ }++ /**+ * Download an HTML page and save it as cleaned HTML or Markdown (scripts+ * stripped, images optionally inlined as data URLs, optional article+ * extraction). Binary URLs should use downloadUrl instead.+ */+ function downloadUrlAs(+ url: string,+ targetPath: string,+ format: 'html' | 'markdown',+ options: {+ extractArticle?: boolean;+ embedImages?: boolean;+ tags?: string;+ } = {},+ ): Promise<TS.FileSystemEntry> {+ const opts: ClipOptions = {+ sourceUrl: url,+ fetchImage: fetchImageDataUrl,+ extractArticle: options.extractArticle,+ embedImages: options.embedImages,+ tags: options.tags,+ scrappedOn: new Date().toISOString(),+ };+ return fetchPageHtml(url)+ .then((html) =>+ format === 'markdown'+ ? htmlToMarkdown(html, opts)+ : htmlToCleanHtml(html, opts),+ )+ .then((content) =>+ saveTextFilePromise({ path: targetPath }, content, true),+ );+ }+function downloadFsEntry(fsEntry: TS.FileSystemEntry) {const loc = findLocation(fsEntry.locationID);if (!loc) {@@ -2689,6 +2859,7 @@ export const IOActionsContextProvider = ({copyDirs,copyFiles,downloadUrl,+ downloadUrlAs,downloadFsEntry,uploadFilesAPI,uploadFiles,diff --git a/src/renderer/locales/en/core.json b/src/renderer/locales/en/core.jsonindex 926cf9cc9..7b96a4d77 100644--- a/src/renderer/locales/en/core.json+++ b/src/renderer/locales/en/core.json@@ -796,6 +796,11 @@"resetFileType": "Reset File Types","override": "Override","saveAs": "Save As",+ "originalFile": "Original file",+ "cleanedHtml": "Cleaned HTML",+ "markdown": "Markdown",+ "extractMainArticle": "Extract main article",+ "embedImages": "Embed images","resolveConflictTitle": "File was modified externally","resolveConflictDesc": "For saving your changes as new file click on 'Save As' button, for override the external changes click the 'Override' button.","create": "Create",diff --git a/src/renderer/perspectives/common/NewSubMenu.tsx b/src/renderer/perspectives/common/NewSubMenu.tsxindex c9507f519..f44a1d095 100644--- a/src/renderer/perspectives/common/NewSubMenu.tsx+++ b/src/renderer/perspectives/common/NewSubMenu.tsx@@ -19,6 +19,7 @@ import AppConfig from '-/AppConfig';import {AddExistingFileIcon,AudioRecordIcon,+ DownloadIcon,HTMLFileIcon,LinkFileIcon,MarkdownFileIcon,@@ -27,7 +28,8 @@ import {SmallArrowRightIcon,TemplateFileIcon,} from '-/components/CommonIcons';-import { ProLabel } from '-/components/HelperComponents';+import { useDownloadUrlDialogContext } from '-/components/dialogs/hooks/useDownloadUrlDialogContext';+import { BetaLabel, ProLabel } from '-/components/HelperComponents';import InfoIcon from '-/components/InfoIcon';import TsMenuList from '-/components/TsMenuList';import { Pro } from '-/pro';@@ -67,6 +69,7 @@ function NewSubMenu(props: Props) {addExistingFile,} = props;const hideProFeatures: boolean = useSelector(isHideProFeatures);+ const { openDownloadUrl } = useDownloadUrlDialogContext();const [anchorEl, setAnchorEl] = useState<HTMLElement | null>(null);// Run a create action and dismiss the whole menu chain.@@ -222,6 +225,23 @@ function NewSubMenu(props: Props) {<ListItemText primary={t('core:addFiles')} /></MenuItem>)}+ <MenuItem+ key="newFromDownloadURL"+ data-tid="newFromDownloadURLTID"+ onClick={() => run(() => openDownloadUrl())}+ >+ <ListItemIcon>+ <DownloadIcon />+ </ListItemIcon>+ <ListItemText+ primary={+ <>+ {t('core:newFromDownloadURL')}+ <BetaLabel />+ </>+ }+ />+ </MenuItem></TsMenuList></Menu></>diff --git a/src/renderer/services/io-capacitor.ts b/src/renderer/services/io-capacitor.tsindex dfbdabfba..d7082400f 100644--- a/src/renderer/services/io-capacitor.ts+++ b/src/renderer/services/io-capacitor.ts@@ -20,7 +20,7 @@ const {const U = require('./capacitor-io-utils');// Capacitor imports-const { Capacitor } = require('@capacitor/core');+const { Capacitor, CapacitorHttp } = require('@capacitor/core');const { Filesystem, Directory, Encoding } = require('@capacitor/filesystem');const { App } = require('@capacitor/app');const { Device } = require('@capacitor/device');@@ -1143,6 +1143,32 @@ function shareFiles(files) {});}+/**+ * Fetch a remote URL over the native HTTP stack. WKWebView rejects cross-origin+ * fetch() from the capacitor:// origin with "Load failed"; CapacitorHttp runs+ * the request natively and isn't bound by CORS. Returns the body as a base64+ * string plus the content-type (the shape the native plugin yields for a binary+ * responseType), leaving byte/data-URL conversion to the caller.+ */+function httpGet(url) {+ return CapacitorHttp.request({+ url,+ method: 'GET',+ responseType: 'arraybuffer',+ }).then((res) => {+ if (res.status < 200 || res.status >= 300) {+ throw new Error('HTTP ' + res.status);+ }+ const contentType =+ (res.headers &&+ (res.headers['content-type'] || res.headers['Content-Type'])) ||+ '';+ // Native returns the binary body as a base64 string.+ const base64 = typeof res.data === 'string' ? res.data : '';+ return { base64, contentType };+ });+}+/*** Download a file under Capacitor. Replaces the Cordova-only* `window.plugins.Downloader`, which is undefined under Capacitor.@@ -1278,5 +1304,6 @@ export {focusWindow,shareFiles,downloadFile,+ httpGet,getNativeFileUrlAsync,};diff --git a/src/renderer/services/web-content.ts b/src/renderer/services/web-content.tsnew file mode 100644index 000000000..aecc1d52f--- /dev/null+++ b/src/renderer/services/web-content.ts@@ -0,0 +1,381 @@+/**+ * TagSpaces - universal file and folder organizer+ * Copyright (C) 2017-present TagSpaces GmbH+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License (version 3) as+ * published by the Free Software Foundation.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <https://www.gnu.org/licenses/>.+ *+ */++/**+ * Platform-agnostic web-content processing ported from the TagSpaces browser+ * extension (../browser-ext: src/ui/popup.js + src/lib/utils.js). It turns a raw+ * HTML page into either sanitized HTML or Markdown, optionally extracting the+ * main article and inlining images as data URLs.+ *+ * All IO is injected (`fetchImage`) so this module stays free of `window`,+ * `electron` or `chrome` references.+ */++import { isProbablyReaderable, Readability } from '@mozilla/readability';+import DOMPurify from 'dompurify';+import TurndownService from 'turndown';++export interface ClipOptions {+ /** Absolute URL of the page, used to resolve relative image/asset URLs. */+ sourceUrl: string;+ /** Returns a `data:` URL for the given absolute image URL, or null on failure. */+ fetchImage: (url: string) => Promise<string | null>;+ /** Run Readability to keep only the main article. */+ extractArticle?: boolean;+ /** Inline referenced images as base64 `data:` URLs. */+ embedImages?: boolean;+ /** Comma-separated tags written into the Markdown front matter. */+ tags?: string;+ /** ISO timestamp stamped into metadata (injected so the package stays pure). */+ scrappedOn?: string;+}++type ImageTask = () => Promise<void>;++const IMG_BATCH_SIZE = 5;+// Cap how many images we inline so a hostile page can't trigger thousands of+// sub-resource fetches / a giant output file.+const MAX_INLINE_IMAGES = 200;++// Only inline images fetched over http/https. Image URLs come from untrusted+// page content, so block file:/data:/intranet schemes from being fetched.+function isHttpUrl(url: string): boolean {+ return /^https?:\/\//i.test(url);+}++// ---------------------------------------------------------------------------+// Front matter helpers (ported from popup.js:914-956)+// ---------------------------------------------------------------------------++export function escapeYamlString(str: string): string {+ return str.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');+}++function formatFrontMatterEntry(key: string, value: string): string {+ if (key === 'tags') {+ const tags = value+ .split(',')+ .map((t) => t.trim())+ .filter(Boolean)+ .map((tag) => ` - "${escapeYamlString(tag)}"\n`)+ .join('');+ return `tags:\n${tags}`;+ }+ return `${key}: "${escapeYamlString(value)}"\n`;+}++/** Inject or update a YAML front-matter block at the top of the content. */+export function injectFrontMatter(+ content: string,+ newMeta: Record<string, string>,+): string {+ const frontMatterRegex = /^---\n([\s\S]*?)\n---/;+ const match = content.match(frontMatterRegex);++ if (match) {+ const existingYaml = match[1];+ const additionalYaml = Object.entries(newMeta)+ .filter(([key]) => !existingYaml.includes(`${key}:`))+ .map(([key, value]) => formatFrontMatterEntry(key, value))+ .join('');+ return content.replace(+ frontMatterRegex,+ `---\n${existingYaml}\n${additionalYaml}---`,+ );+ }++ const yamlBlock = Object.entries(newMeta)+ .map(([key, value]) => formatFrontMatterEntry(key, value))+ .join('');+ return `---\n${yamlBlock}---\n\n${content}`;+}++// ---------------------------------------------------------------------------+// Image helpers (ported from utils.js:20-61 + popup.js batching)+// ---------------------------------------------------------------------------++/** Robustly parse srcset and pick the highest-resolution URL (utils.js:20). */+export function getHighestResUrl(el: Element, baseUrl: string): string | null {+ const src = el.getAttribute('src');+ const srcset = el.getAttribute('srcset');+ const candidates: { url: string; value: number; type: 'w' | 'x' }[] = [];++ if (src) {+ candidates.push({ url: src, value: 1, type: 'x' });+ }++ if (srcset) {+ const regex = /([^,\s]+)\s*(?:(\d+)w|(\d+(?:\.\d+)?)x)?/g;+ Array.from(srcset.matchAll(regex)).forEach((match) => {+ const url = match[1];+ if (!url) {+ return;+ }+ const wDesc = match[2] ? parseInt(match[2], 10) : null;+ const xDesc = match[3] ? parseFloat(match[3]) : null;+ if (wDesc) {+ candidates.push({ url, value: wDesc, type: 'w' });+ } else if (xDesc) {+ candidates.push({ url, value: xDesc, type: 'x' });+ } else {+ candidates.push({ url, value: 1, type: 'x' });+ }+ });+ }++ if (candidates.length === 0) {+ return null;+ }++ // Rank candidates: prefer width ('w') descriptors over density ('x').+ const best = candidates.reduce((prev, curr) => {+ if (curr.type === 'w' && prev.type === 'w') {+ return curr.value > prev.value ? curr : prev;+ }+ if (curr.type === 'w' && prev.type === 'x') {+ return curr;+ }+ if (curr.type === 'x' && prev.type === 'w') {+ return prev;+ }+ return curr.value > prev.value ? curr : prev;+ });++ try {+ return new URL(best.url, baseUrl).href;+ } catch (e) {+ return null;+ }+}++async function processInBatches(+ tasks: ImageTask[],+ batchSize: number,+): Promise<void> {+ for (let i = 0; i < tasks.length; i += batchSize) {+ const batch = tasks.slice(i, i + batchSize);+ // eslint-disable-next-line no-await-in-loop+ await Promise.all(batch.map((task) => task()));+ }+}++/** Inline `<img>`/`<video poster>` sources of a document as data URLs in place. */+async function inlineImagesInDoc(+ doc: Document,+ opts: ClipOptions,+): Promise<void> {+ const targets: { selector: string; attribute: string }[] = [+ { selector: 'img', attribute: 'src' },+ { selector: 'video[poster]', attribute: 'poster' },+ ];++ const tasks: ImageTask[] = targets.flatMap((target) =>+ Array.from(doc.querySelectorAll(target.selector))+ .map((el): ImageTask | null => {+ const originalUrl = el.getAttribute(target.attribute);+ if (!originalUrl || originalUrl.startsWith('data:')) {+ return null;+ }+ let absoluteUrl: string | null;+ try {+ absoluteUrl =+ target.selector === 'img'+ ? getHighestResUrl(el, opts.sourceUrl)+ : new URL(originalUrl, opts.sourceUrl).href;+ } catch (e) {+ return null;+ }+ if (!absoluteUrl || !isHttpUrl(absoluteUrl)) {+ return null;+ }+ const url = absoluteUrl;+ return async () => {+ const dataUrl = await opts.fetchImage(url);+ if (dataUrl && dataUrl.startsWith('data:image')) {+ el.setAttribute(target.attribute, dataUrl);+ el.removeAttribute('srcset');+ }+ };+ })+ .filter((task): task is ImageTask => task !== null),+ );++ await processInBatches(tasks.slice(0, MAX_INLINE_IMAGES), IMG_BATCH_SIZE);+}++/** Inline images referenced inside Markdown as data URLs (popup.js:824). */+async function inlineImagesInMarkdown(+ markdown: string,+ opts: ClipOptions,+): Promise<string> {+ const regexTargets = [+ { regex: /!\[(.*?)\]\(((?!data:).+?)(?:\s+"(.*?)")?\)/g, urlIndex: 2 },+ { regex: /<img\s+[^>]*src=["']((?!data:)[^"']+)["'][^>]*>/g, urlIndex: 1 },+ ];+ const replacements = new Map<string, string | null>();+ const tasks: ImageTask[] = [];++ regexTargets.forEach((target) => {+ Array.from(markdown.matchAll(target.regex)).forEach((match) => {+ const originalUrl = match[target.urlIndex];+ if (!originalUrl || replacements.has(originalUrl)) {+ return;+ }+ let absoluteUrl: string;+ try {+ absoluteUrl = new URL(originalUrl, opts.sourceUrl).href;+ } catch (e) {+ return;+ }+ if (!isHttpUrl(absoluteUrl)) {+ return;+ }+ replacements.set(originalUrl, null);+ tasks.push(async () => {+ const dataUrl = await opts.fetchImage(absoluteUrl);+ if (dataUrl && dataUrl.startsWith('data:image')) {+ replacements.set(originalUrl, dataUrl);+ } else {+ replacements.delete(originalUrl);+ }+ });+ });+ });++ await processInBatches(tasks.slice(0, MAX_INLINE_IMAGES), IMG_BATCH_SIZE);++ let result = markdown;+ Array.from(replacements.entries()).forEach(([originalUrl, dataUrl]) => {+ if (dataUrl) {+ const escapedUrl = originalUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ result = result.replace(new RegExp(escapedUrl, 'g'), dataUrl);+ }+ });+ return result;+}++// ---------------------------------------------------------------------------+// Core sanitize / extract pipeline+// ---------------------------------------------------------------------------++/**+ * Parse raw HTML, optionally reduce it to the Readability article, and sanitize+ * it (DOMPurify strips `<script>` and inline event handlers). Returns a fresh+ * sanitized Document.+ */+function buildCleanDoc(html: string, opts: ClipOptions): Document {+ const parser = new DOMParser();+ let workingHtml = html;++ if (opts.extractArticle) {+ try {+ const doc = parser.parseFromString(html, 'text/html');+ if (isProbablyReaderable(doc)) {+ const article = new Readability(doc).parse();+ if (article && article.content) {+ const title = article.title+ ? `<h1>${article.title+ .replace(/&/g, '&')+ .replace(/</g, '<')+ .replace(/>/g, '>')}</h1>\n`+ : '';+ workingHtml = title + article.content;+ }+ }+ } catch (e) {+ // fall back to the full page on any Readability failure+ }+ }++ const clean = DOMPurify.sanitize(workingHtml, { WHOLE_DOCUMENT: true });+ return parser.parseFromString(clean, 'text/html');+}++// ---------------------------------------------------------------------------+// Public API+// ---------------------------------------------------------------------------++/** Produce sanitized, self-contained HTML with embedded source metadata. */+export async function htmlToCleanHtml(+ html: string,+ opts: ClipOptions,+): Promise<string> {+ const doc = buildCleanDoc(html, opts);++ if (opts.embedImages !== false) {+ await inlineImagesInDoc(doc, opts);+ }++ let { head } = doc;+ if (!head) {+ head = doc.createElement('head');+ doc.documentElement.insertBefore(head, doc.body);+ }+ if (!head.querySelector('meta[charset]')) {+ const charsetMeta = doc.createElement('meta');+ charsetMeta.setAttribute('charset', 'utf-8');+ head.insertBefore(charsetMeta, head.firstChild);+ }++ const { body } = doc;+ if (body) {+ body.setAttribute('data-createdwith', 'TagSpaces');+ body.setAttribute('data-sourceurl', opts.sourceUrl);+ if (opts.scrappedOn) {+ body.setAttribute('data-scrappedon', opts.scrappedOn);+ }+ }++ const doctype = doc.doctype+ ? `${new XMLSerializer().serializeToString(doc.doctype)}\n`+ : '<!DOCTYPE html>\n';+ return doctype + doc.documentElement.outerHTML;+}++/** Convert the (optionally article-extracted) page to Markdown with front matter. */+export async function htmlToMarkdown(+ html: string,+ opts: ClipOptions,+): Promise<string> {+ const doc = buildCleanDoc(html, opts);+ const turndownService = new TurndownService({+ headingStyle: 'atx',+ codeBlockStyle: 'fenced',+ fence: '```',+ });++ let markdown = '';+ try {+ markdown = turndownService.turndown(doc.body ? doc.body.innerHTML : '');+ } catch (e) {+ markdown = '';+ }++ if (opts.embedImages !== false) {+ markdown = await inlineImagesInMarkdown(markdown, opts);+ }++ const metadata: Record<string, string> = {+ url: opts.sourceUrl,+ date: opts.scrappedOn || '',+ };+ if (opts.tags && opts.tags.trim()) {+ metadata.tags = opts.tags.trim();+ }+ return injectFrontMatter(markdown, metadata);+}diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.jsnew file mode 100644index 000000000..09a871624--- /dev/null+++ b/tests/e2e/download-url.pw.e2e.js@@ -0,0 +1,147 @@+/*+ * Copyright (c) 2016-present - TagSpaces GmbH. All rights reserved.+ */+import fs from 'fs';+import http from 'http';+import path from 'path';++import { expect, test } from './fixtures';+import {+ clickOn,+ clickOnIfVisible,+ expectElementExist,+ getGridFileSelector,+ reloadDirectory,+ setInputValue,+} from './general.helpers';+import { startTestingApp, stopApp } from './hook';+import {+ createPwLocation,+ createS3Location,+ defaultLocationName,+} from './location.helpers';+import { clickOnMenuOperation } from './test-utils';+import { clearDataStorage, closeWelcomePlaywright } from './welcome.helpers';++// A tiny localhost HTTP server stands in for an arbitrary remote site so the+// download path is exercised end-to-end without network flakiness. The Electron+// main process reaches it over net.fetch the same way it would a real URL.+const SAMPLE_TXT = 'TagSpaces download URL test content';+const PAGE_HTML = `<!DOCTYPE html>+<html>+ <head><title>Clip Test Page</title></head>+ <body>+ <h1>Hello Heading</h1>+ <p>Some paragraph text for clipping.</p>+ <script>window.__evil = 'should-be-stripped';</script>+ </body>+</html>`;++let server;+let baseUrl;++async function openDownloadUrlDialog() {+ await clickOn('[data-tid=folderContainerOpenDirMenu]');+ await clickOnMenuOperation('newFromDownloadURLTID');+ await global.client.waitForSelector('[data-tid=downloadUrlDialogTID]', {+ state: 'visible',+ timeout: 8000,+ });+}++test.beforeAll(async ({ isWeb, isS3, webServerPort }, testInfo) => {+ server = http.createServer((req, res) => {+ if (req.url === '/sample.txt') {+ res.writeHead(200, { 'Content-Type': 'text/plain' });+ res.end(SAMPLE_TXT);+ } else if (req.url === '/page.html') {+ res.writeHead(200, { 'Content-Type': 'text/html' });+ res.end(PAGE_HTML);+ } else {+ res.writeHead(404);+ res.end('not found');+ }+ });+ await new Promise((resolve) => {+ server.listen(0, '127.0.0.1', resolve);+ });+ baseUrl = `http://127.0.0.1:${server.address().port}`;++ if (isS3) {+ await startTestingApp({ isWeb, isS3, webServerPort, testInfo });+ await closeWelcomePlaywright();+ } else {+ await startTestingApp(+ { isWeb, isS3, webServerPort, testInfo },+ 'extconfig.js',+ );+ }+});++test.afterAll(async () => {+ await stopApp();+ if (server) {+ await new Promise((resolve) => server.close(resolve));+ }+});++test.afterEach(async ({ isS3, testDataDir }) => {+ // Keep the shared local test data clean (S3 is reset by other suites' refresh).+ if (!isS3) {+ ['sample.txt', 'page.md', 'page.html'].forEach((name) => {+ const p = path.join(testDataDir, name);+ if (fs.existsSync(p)) {+ fs.rmSync(p, { force: true });+ }+ });+ }+ await clearDataStorage();+});++test.beforeEach(async ({ isS3, testDataDir }) => {+ if (isS3) {+ await createS3Location('', defaultLocationName, true);+ await closeWelcomePlaywright();+ } else {+ await createPwLocation(testDataDir, defaultLocationName, true);+ }+ await clickOn('[data-tid=location_' + defaultLocationName + ']');+ await expectElementExist(getGridFileSelector('empty_folder'), true, 15000);+});++test.describe('TST54 - Download from URL', () => {+ test('TST5420 - Download raw file into the location [electron,s3]', async () => {+ await openDownloadUrlDialog();+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/sample.txt');+ // Default "Original file" format → straight binary download into the location.+ await clickOn('[data-tid=downloadFileUrlTID]');+ await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);+ // The raw download opens the upload-progress dialog — dismiss it before+ // interacting with the directory menu again.+ await clickOnIfVisible('[data-tid=closeFileUploadTID]');+ await reloadDirectory();+ await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);+ });++ test('TST5421 - Clip an HTML page as Markdown [electron,s3]', async ({+ isS3,+ testDataDir,+ }) => {+ await openDownloadUrlDialog();+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');+ await clickOn('[data-tid=downloadFormatMarkdownTID]');+ await clickOn('[data-tid=downloadFileUrlTID]');+ // page.html → page.md saved into the location.+ await expectElementExist(getGridFileSelector('page.md'), true, 20000);++ // On local locations assert the conversion result on disk: heading/text+ // survive, the <script> is stripped by DOMPurify.+ if (!isS3) {+ const md = fs.readFileSync(path.join(testDataDir, 'page.md'), 'utf8');+ expect(md).toContain('Hello Heading');+ expect(md).toContain('Some paragraph text');+ expect(md).not.toContain('should-be-stripped');+ expect(md).not.toContain('<script');+ }+ });+});diff --git a/tests/e2e/test-utils.js b/tests/e2e/test-utils.jsindex ab1b809bf..2858e6a02 100644--- a/tests/e2e/test-utils.js+++ b/tests/e2e/test-utils.js@@ -23,6 +23,7 @@ const submenuParentTid = {createNewAudioTID: 'createNewSubmenuTID',newSubDirectory: 'createNewSubmenuTID',addExistingFile: 'createNewSubmenuTID',+ newFromDownloadURLTID: 'createNewSubmenuTID',};// Resolve the parent submenu trigger for a menu operation, if it lives in a