Skip to content

TagSpaces is an offline, open source, document manager with tagging support

feat: added url scrapping functionality

Ilian Sapundshievd4601037bce4
diff --git a/package-lock.json b/package-lock.json
index 5da02fb67..116f212a6 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -26,6 +26,7 @@
"@milkdown/crepe": "7.21.2",
"@milkdown/kit": "7.21.2",
"@milkdown/react": "7.21.2",
+ "@mozilla/readability": "0.6.0",
"@mui/icons-material": "9.1.0",
"@mui/lab": "9.0.0-beta.4",
"@mui/material": "9.1.0",
@@ -86,6 +87,7 @@
"semver": "7.7.4",
"swiper": "12.1.3",
"tga-js": "1.1.1",
+ "turndown": "7.2.4",
"utif.ts": "3.1.1",
"zod": "3.25.67",
"zod-to-json-schema": "3.25.1"
@@ -103,6 +105,7 @@
"@types/react-redux": "^7.1.24",
"@types/redux-logger": "^3.0.9",
"@types/terser-webpack-plugin": "^5.0.4",
+ "@types/turndown": "^5.0.5",
"@types/webpack-bundle-analyzer": "^4.6.0",
"@typescript-eslint/eslint-plugin": "^6.7.0",
"@typescript-eslint/parser": "^6.7.0",
@@ -6018,6 +6021,21 @@
"nanoid": "^5.0.9"
}
},
+ "node_modules/@mixmark-io/domino": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
+ "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
+ "license": "BSD-2-Clause"
+ },
+ "node_modules/@mozilla/readability": {
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
+ "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=14.0.0"
+ }
+ },
"node_modules/@mui/core-downloads-tracker": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-9.1.0.tgz",
@@ -9906,6 +9924,13 @@
"license": "MIT",
"optional": true
},
+ "node_modules/@types/turndown": {
+ "version": "5.0.5",
+ "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.5.tgz",
+ "integrity": "sha512-TL2IgGgc7B5j78rIccBtlYAnkuv8nUQqhQc+DSYV5j9Be9XOcm/SKOVRuA47xAVI3680Tk9B1d8flK2GWT2+4w==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/@types/unist": {
"version": "3.0.3",
"license": "MIT"
@@ -29290,6 +29315,19 @@
"node": "^20.17.0 || >=22.9.0"
}
},
+ "node_modules/turndown": {
+ "version": "7.2.4",
+ "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",
+ "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@mixmark-io/domino": "^2.2.0"
+ },
+ "engines": {
+ "node": ">=18",
+ "npm": ">=9"
+ }
+ },
"node_modules/type-check": {
"version": "0.4.0",
"dev": true,
diff --git a/package.json b/package.json
index 4dba975d5..bfccb87c9 100644
--- a/package.json
+++ b/package.json
@@ -184,6 +184,7 @@
"@milkdown/crepe": "7.21.2",
"@milkdown/kit": "7.21.2",
"@milkdown/react": "7.21.2",
+ "@mozilla/readability": "0.6.0",
"@mui/icons-material": "9.1.0",
"@mui/lab": "9.0.0-beta.4",
"@mui/material": "9.1.0",
@@ -244,6 +245,7 @@
"semver": "7.7.4",
"swiper": "12.1.3",
"tga-js": "1.1.1",
+ "turndown": "7.2.4",
"utif.ts": "3.1.1",
"zod": "3.25.67",
"zod-to-json-schema": "3.25.1"
@@ -264,6 +266,7 @@
"@types/react-redux": "^7.1.24",
"@types/redux-logger": "^3.0.9",
"@types/terser-webpack-plugin": "^5.0.4",
+ "@types/turndown": "^5.0.5",
"@types/webpack-bundle-analyzer": "^4.6.0",
"@typescript-eslint/eslint-plugin": "^6.7.0",
"@typescript-eslint/parser": "^6.7.0",
diff --git a/src/main/mainEvents.ts b/src/main/mainEvents.ts
index 24145e88c..feb3bc165 100644
--- a/src/main/mainEvents.ts
+++ b/src/main/mainEvents.ts
@@ -30,9 +30,8 @@ import {
isDirectory,
} from '@tagspaces/tagspaces-common-node/io-node';
import fs from 'fs-extra';
-import http from 'http';
-import https from 'https';
import path from 'path';
+import { Readable } from 'stream';
import WebSocket from 'ws';
import {
getOnProgress,
@@ -49,6 +48,37 @@ import registerSecureStorageEvents from './secureStorage';
//let watcher: FSWatcher;
const progress = {};
+
+// A desktop-Chrome User-Agent so remote servers treat our downloads like a
+// regular browser request. Many sites reject the default Electron/Node agent
+// (403 "non-browser host"), which is why direct downloads previously failed.
+const BROWSER_USER_AGENT =
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+
+function browserHeaders(url: string): Record<string, string> {
+ const headers: Record<string, string> = {
+ 'User-Agent': BROWSER_USER_AGENT,
+ Accept: '*/*',
+ };
+ try {
+ headers.Referer = new URL(url).origin;
+ } catch (e) {
+ // invalid URL — net.fetch will reject below
+ }
+ return headers;
+}
+
+// Only allow http/https downloads. Without this, net.fetch would also resolve
+// file:// (local file read) and other schemes — an SSRF / local-file-read risk,
+// especially since image URLs are taken from fetched (untrusted) page content.
+function isHttpUrl(url: string): boolean {
+ try {
+ const { protocol } = new URL(url);
+ return protocol === 'http:' || protocol === 'https:';
+ } catch (e) {
+ return false;
+ }
+}
let wsc;
const controllers = new Map(); // requestId -> AbortController
@@ -112,59 +142,104 @@ export default function loadMainEvents() {
return `data:${contentType};base64,${base64}`;
});
- ipcMain.handle('fetchUrl', async (event, url, targetPath, withProgress) => {
- function fetchHttp(url) {
- return new Promise((resolve, reject) => {
- const client = url.startsWith('https') ? https : http;
-
- client
- .get(url, (res) => {
- let downloadedSize = 0;
- const totalSize = parseInt(res.headers['content-length'], 10);
-
- let onUploadProgress = undefined;
- if (withProgress) {
- progress['fetchUrl'] = newProgress('fetchUrl', totalSize);
- onUploadProgress = getOnProgress('fetchUrl', progress);
- }
-
- const fileStream = fs.createWriteStream(targetPath); // Write file to disk
- // Pipe the response data into the file
- res.pipe(fileStream);
+ // Download a URL straight to disk (local locations). Uses Electron's net.fetch
+ // (Chromium network stack — shares session/cookies, follows redirects, honors
+ // proxy/gzip, and is not bound by the renderer's CSP or the server's CORS) with
+ // browser-like headers so servers don't block us as a non-browser host.
+ ipcMain.handle('fetchUrl', async (_event, url, targetPath, withProgress) => {
+ if (!isHttpUrl(url)) {
+ return { error: 'Unsupported URL scheme (only http/https allowed)' };
+ }
+ try {
+ const response = await net.fetch(url, { headers: browserHeaders(url) });
+ if (!response.ok) {
+ return { error: `${response.status} ${response.statusText}` };
+ }
+ const contentType = response.headers.get('content-type') || '';
+ const totalSize = parseInt(
+ response.headers.get('content-length') || '',
+ 10,
+ );
- // Collect data chunks and calculate size
- res.on('data', (chunk) => {
- downloadedSize += chunk.length; // Add the size of the current chunk
- if (onUploadProgress) {
- onUploadProgress(
- { key: targetPath, loaded: downloadedSize, total: totalSize },
- undefined,
- );
- }
- });
+ let onUploadProgress;
+ if (withProgress) {
+ progress['fetchUrl'] = newProgress('fetchUrl', totalSize);
+ onUploadProgress = getOnProgress('fetchUrl', progress);
+ }
- // Resolve the response on end
- res.on('end', () => {
- if (onUploadProgress) {
- onUploadProgress(
- { key: targetPath, loaded: 1, total: 1 },
- undefined,
- );
- }
- resolve({ success: true, filePath: targetPath });
- });
- })
- .on('error', (err) => {
- reject(err);
- });
+ await new Promise<void>((resolve, reject) => {
+ const fileStream = fs.createWriteStream(targetPath);
+ const nodeStream = Readable.fromWeb(response.body as any);
+ let downloadedSize = 0;
+ nodeStream.on('data', (chunk) => {
+ downloadedSize += chunk.length;
+ if (onUploadProgress) {
+ onUploadProgress(
+ { key: targetPath, loaded: downloadedSize, total: totalSize },
+ undefined,
+ );
+ }
+ });
+ nodeStream.on('error', reject);
+ fileStream.on('error', reject);
+ fileStream.on('finish', resolve);
+ nodeStream.pipe(fileStream);
});
+
+ if (onUploadProgress) {
+ onUploadProgress({ key: targetPath, loaded: 1, total: 1 }, undefined);
+ }
+ return { success: true, filePath: targetPath, contentType };
+ } catch (error) {
+ return { error: error.message };
}
+ });
+ // Fetch a URL and return its bytes (object-store locations + image inlining for
+ // the HTML/Markdown "save as" formats). Same browser-stack request as fetchUrl.
+ // `options` lets callers harden untrusted sub-resource fetches (page images):
+ // omitCredentials — don't send the app session's cookies to the target
+ // maxBytes — reject oversized responses (memory/disk DoS guard)
+ // timeoutMs — abort hung requests
+ ipcMain.handle('fetchUrlBuffer', async (_event, url, options = {}) => {
+ if (!isHttpUrl(url)) {
+ return { error: 'Unsupported URL scheme (only http/https allowed)' };
+ }
+ const { omitCredentials, maxBytes, timeoutMs } = options;
+ const controller = new AbortController();
+ const timer = timeoutMs
+ ? setTimeout(() => controller.abort(), timeoutMs)
+ : undefined;
try {
- const response = await fetchHttp(url);
- return response;
+ const response = await net.fetch(url, {
+ headers: browserHeaders(url),
+ credentials: omitCredentials ? 'omit' : undefined,
+ signal: controller.signal,
+ });
+ if (!response.ok) {
+ return { error: `${response.status} ${response.statusText}` };
+ }
+ if (maxBytes) {
+ const declared = parseInt(
+ response.headers.get('content-length') || '',
+ 10,
+ );
+ if (Number.isFinite(declared) && declared > maxBytes) {
+ return { error: 'Response too large' };
+ }
+ }
+ const contentType = response.headers.get('content-type') || '';
+ const data = await response.arrayBuffer();
+ if (maxBytes && data.byteLength > maxBytes) {
+ return { error: 'Response too large' };
+ }
+ return { data, contentType };
} catch (error) {
return { error: error.message };
+ } finally {
+ if (timer) {
+ clearTimeout(timer);
+ }
}
});
ipcMain.handle('isWorkerAvailable', async () => {
diff --git a/src/main/preload.ts b/src/main/preload.ts
index 14973bcd6..9df64eb1e 100644
--- a/src/main/preload.ts
+++ b/src/main/preload.ts
@@ -10,6 +10,7 @@ import {
export type Channels =
| 'isWorkerAvailable'
| 'fetchUrl'
+ | 'fetchUrlBuffer'
| 'isDirectory'
| 'resolveRelativePaths'
| 'set-language'
diff --git a/src/renderer/components/MobileNavigation.tsx b/src/renderer/components/MobileNavigation.tsx
index 9db98280b..88445302d 100644
--- a/src/renderer/components/MobileNavigation.tsx
+++ b/src/renderer/components/MobileNavigation.tsx
@@ -42,7 +42,7 @@ import {
} from '-/components/CommonIcons';
import CustomLogo from '-/components/CustomLogo';
import HelpFeedbackPanel from '-/components/HelpFeedbackPanel';
-import { ProLabel } from '-/components/HelperComponents';
+import { BetaLabel, ProLabel } from '-/components/HelperComponents';
import InfoIcon from '-/components/InfoIcon';
import LocationManager from '-/components/LocationManager';
import ProTeaser from '-/components/ProTeaser';
@@ -511,25 +511,27 @@ function MobileNavigation(props: Props) {
</ListItemIcon>
<ListItemText primary={t('core:addFiles')} />
</MenuItem>
- {AppConfig.isElectron &&
- !currentLocation?.haveObjectStoreSupport() && (
- <MenuItem
- key="newFromDownloadURL"
- data-tid="newFromDownloadURLTID"
- onClick={() => {
- openDownloadUrl();
- setOpenCreateMenu(false);
- hideDrawer?.();
- }}
- >
- <ListItemIcon>
- <DownloadIcon />
- </ListItemIcon>
- <ListItemText
- primary={t('core:newFromDownloadURL')}
- />
- </MenuItem>
- )}
+ <MenuItem
+ key="newFromDownloadURL"
+ data-tid="newFromDownloadURLTID"
+ onClick={() => {
+ openDownloadUrl();
+ setOpenCreateMenu(false);
+ hideDrawer?.();
+ }}
+ >
+ <ListItemIcon>
+ <DownloadIcon />
+ </ListItemIcon>
+ <ListItemText
+ primary={
+ <>
+ {t('core:newFromDownloadURL')}
+ <BetaLabel />
+ </>
+ }
+ />
+ </MenuItem>
<Divider />
<MenuItem
key="createNewFolder"
diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
index 73c7c6107..aeb756df2 100644
--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx
+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
@@ -20,18 +20,25 @@ import AppConfig from '-/AppConfig';
import DraggablePaper from '-/components/DraggablePaper';
import TsButton from '-/components/TsButton';
import TsTextField from '-/components/TsTextField';
+import TsToggleButton from '-/components/TsToggleButton';
import TargetPath from '-/components/dialogs/components/TargetPath';
import TsDialogActions from '-/components/dialogs/components/TsDialogActions';
import TsDialogTitle from '-/components/dialogs/components/TsDialogTitle';
import { TargetPathContextProvider } from '-/components/dialogs/hooks/TargetPathContextProvider';
import { useFileUploadDialogContext } from '-/components/dialogs/hooks/useFileUploadDialogContext';
import { useTargetPathContext } from '-/components/dialogs/hooks/useTargetPathContext';
-import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';
import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';
import { useIOActionsContext } from '-/hooks/useIOActionsContext';
import { useNotificationContext } from '-/hooks/useNotificationContext';
import { actions as AppActions, AppDispatch } from '-/reducers/app';
import { TS } from '-/tagspaces.namespace';
+import {
+ Box,
+ Checkbox,
+ FormControlLabel,
+ ToggleButtonGroup,
+ Typography,
+} from '@mui/material';
import Dialog from '@mui/material/Dialog';
import DialogContent from '@mui/material/DialogContent';
import Paper from '@mui/material/Paper';
@@ -47,22 +54,47 @@ interface Props {
onClose: (event?: Object, reason?: string) => void;
}
+// Derive a safe, single-segment filename from a URL-derived name. Decoding the
+// raw name can reintroduce path separators / `..` (e.g. "%2e%2e%2f"), so keep
+// only the final segment and drop traversal/control characters to prevent
+// writing outside the chosen target directory.
+function sanitizeDownloadFileName(name: string): string {
+ let decoded = name;
+ try {
+ decoded = decodeURIComponent(name);
+ } catch (e) {
+ // keep the raw name if it isn't valid percent-encoding
+ }
+ const base = decoded.split(/[/\\]/).pop() || '';
+ const cleaned = Array.from(base)
+ .filter((ch) => ch.charCodeAt(0) >= 32)
+ .join('');
+ return /^\.+$/.test(cleaned) ? '' : cleaned;
+}
+
function DownloadUrlDialog(props: Props) {
const { open, onClose } = props;
const { t } = useTranslation();
const theme = useTheme();
const smallScreen = useMediaQuery(theme.breakpoints.down('md'));
- const { currentLocationId, findLocation, getFirstRWLocation } =
- useCurrentLocationContext();
const { setReflectActions } = useEditedEntryContext();
- const { downloadUrl } = useIOActionsContext();
+ const { downloadUrl, downloadUrlAs } = useIOActionsContext();
const { showNotification } = useNotificationContext();
const { openFileUploadDialog } = useFileUploadDialogContext();
const dispatch: AppDispatch = useDispatch();
+ // Cleaned-HTML / Markdown conversion needs CORS/CSP-free fetching of the page
+ // and its images — only the desktop and native-mobile builds can do that.
+ const canConvert = AppConfig.isElectron || AppConfig.isNativeMobile;
+
const { targetDirectoryPath } = useTargetPathContext();
const fileUrl = useRef<string>();
const [invalidURL, setInvalidURL] = useState<boolean>(false);
+ const [saveFormat, setSaveFormat] = useState<
+ 'original' | 'html' | 'markdown'
+ >('original');
+ const [extractArticle, setExtractArticle] = useState<boolean>(false);
+ const [embedImages, setEmbedImages] = useState<boolean>(true);
const onUploadProgress = (progress, abort, fileName) => {
dispatch(AppActions.onUploadProgress(progress, abort, fileName));
@@ -74,9 +106,6 @@ function DownloadUrlDialog(props: Props) {
function downloadURL() {
if (fileUrl.current) {
- const location = currentLocationId
- ? findLocation()
- : getFirstRWLocation();
try {
const url = new URL(fileUrl.current);
if (invalidURL) {
@@ -96,51 +125,67 @@ function DownloadUrlDialog(props: Props) {
}
}
if (!fileName) {
- fileName =
+ fileName = `${
url.hostname +
- (pathParts && pathParts.length > 0 ? pathParts.join('-') : '') +
- '.html';
+ (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')
+ }.html`;
} else if (fileName.indexOf('.') === -1) {
- fileName = url.hostname + '-' + fileName + '.html';
+ fileName = `${url.hostname}-${fileName}.html`;
}
- if (!location.haveObjectStoreSupport() && AppConfig.isElectron) {
+
+ const reflectOpen = (entry: TS.FileSystemEntry) => {
+ const reflectAction: TS.EditAction = {
+ action: 'add',
+ entry,
+ open: true,
+ source: 'upload',
+ };
+ setReflectActions(reflectAction);
+ };
+ const notifyError = (e: Error) => {
+ console.log('downloadFile error:', e);
+ dispatch(
+ AppActions.setProgress(fileUrl.current, -1, t('core:errorCORS')),
+ );
+ showNotification(
+ t('core:downloadFileError', { message: e.message }),
+ 'error',
+ true,
+ );
+ };
+
+ const safeFileName =
+ sanitizeDownloadFileName(fileName) || 'download.html';
+
+ if (canConvert && saveFormat !== 'original') {
+ // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline
+ // images and optionally extract the article, then save into the location.
+ const baseName = safeFileName.replace(/\.[^.]+$/, '') || 'download';
+ const ext = saveFormat === 'markdown' ? '.md' : '.html';
+ downloadUrlAs(
+ fileUrl.current,
+ `${targetDirectoryPath}/${baseName}${ext}`,
+ saveFormat,
+ { extractArticle, embedImages },
+ )
+ .then(reflectOpen)
+ .catch(notifyError);
+ } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {
+ // Raw download straight into the location (local + cloud + mobile),
+ // routed through the platform's CORS/CSP-free fetch.
dispatch(AppActions.resetProgress());
openFileUploadDialog();
downloadUrl(
fileUrl.current,
- targetDirectoryPath + '/' + decodeURIComponent(fileName),
+ `${targetDirectoryPath}/${safeFileName}`,
onUploadProgress,
)
- .then((entry) => {
- /*if (location?.haveObjectStoreSupport()) {
- // currently objectStore location in downloadFile use saveFilePromise and this function not have progress handling
- dispatch(AppActions.setProgress(fileUrl.current, 100));
- }*/
- const reflectAction: TS.EditAction = {
- action: 'add',
- entry: entry,
- open: true,
- source: 'upload',
- };
- setReflectActions(reflectAction);
- })
- .catch((e) => {
- console.log('downloadFile error:', e);
- dispatch(
- AppActions.setProgress(
- fileUrl.current,
- -1,
- t('core:errorCORS'),
- ),
- );
- showNotification(
- t('core:downloadFileError', { message: e.message }),
- 'error',
- true,
- );
- });
+ .then(reflectOpen)
+ .catch(notifyError);
} else {
- saveAs(fileUrl.current, decodeURIComponent(fileName));
+ // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain
+ // browser download.
+ saveAs(fileUrl.current, safeFileName);
}
onClose();
} catch (ex) {
@@ -153,7 +198,7 @@ function DownloadUrlDialog(props: Props) {
const okButton = (
<TsButton
variant="contained"
- data-tid={'downloadFileUrlTID'}
+ data-tid="downloadFileUrlTID"
onClick={() => downloadURL()}
sx={
{
@@ -204,6 +249,68 @@ function DownloadUrlDialog(props: Props) {
onChange={handleUrlChange}
/>
<TargetPath />
+ {canConvert && (
+ <Box sx={{ mt: 2 }}>
+ <Typography
+ variant="caption"
+ color="textSecondary"
+ sx={{ display: 'block', mb: 0.5 }}
+ >
+ {t('core:saveAs')}
+ </Typography>
+ <ToggleButtonGroup
+ size="small"
+ value={saveFormat}
+ exclusive
+ onChange={(_, v) => {
+ if (v) setSaveFormat(v);
+ }}
+ data-tid="downloadFormatToggleTID"
+ >
+ <TsToggleButton
+ value="original"
+ data-tid="downloadFormatOriginalTID"
+ >
+ {t('core:originalFile')}
+ </TsToggleButton>
+ <TsToggleButton value="html" data-tid="downloadFormatHtmlTID">
+ {t('core:cleanedHtml')}
+ </TsToggleButton>
+ <TsToggleButton
+ value="markdown"
+ data-tid="downloadFormatMarkdownTID"
+ >
+ {t('core:markdown')}
+ </TsToggleButton>
+ </ToggleButtonGroup>
+ {saveFormat !== 'original' && (
+ <Box sx={{ mt: 1 }}>
+ <FormControlLabel
+ control={
+ <Checkbox
+ size="small"
+ checked={extractArticle}
+ onChange={(e) => setExtractArticle(e.target.checked)}
+ data-tid="downloadExtractArticleTID"
+ />
+ }
+ label={t('core:extractMainArticle')}
+ />
+ <FormControlLabel
+ control={
+ <Checkbox
+ size="small"
+ checked={embedImages}
+ onChange={(e) => setEmbedImages(e.target.checked)}
+ data-tid="downloadEmbedImagesTID"
+ />
+ }
+ label={t('core:embedImages')}
+ />
+ </Box>
+ )}
+ </Box>
+ )}
</DialogContent>
{!smallScreen && (
<TsDialogActions>
diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsx
index fbf7c4e84..0575678a9 100644
--- a/src/renderer/hooks/IOActionsContextProvider.tsx
+++ b/src/renderer/hooks/IOActionsContextProvider.tsx
@@ -48,6 +48,11 @@ import {
openFileMessage,
runConcurrent,
} from '-/services/utils-io';
+import {
+ ClipOptions,
+ htmlToCleanHtml,
+ htmlToMarkdown,
+} from '-/services/web-content';
import { TS } from '-/tagspaces.namespace';
import { CommonLocation } from '-/utils/CommonLocation';
import { base64ToUint8Array } from '-/utils/dom';
@@ -126,6 +131,16 @@ type IOActionsContextData = {
targetPath: string,
onDownloadProgress?: (progress, abort, fileName?) => void,
) => Promise<TS.FileSystemEntry>;
+ downloadUrlAs: (
+ url: string,
+ targetPath: string,
+ format: 'html' | 'markdown',
+ options?: {
+ extractArticle?: boolean;
+ embedImages?: boolean;
+ tags?: string;
+ },
+ ) => Promise<TS.FileSystemEntry>;
downloadFsEntry: (fsEntry: TS.FileSystemEntry) => void;
uploadFilesAPI: (
files: Array<any>,
@@ -259,6 +274,7 @@ export const IOActionsContext = createContext<IOActionsContextData>({
copyDirs: undefined,
copyFiles: undefined,
downloadUrl: undefined,
+ downloadUrlAs: undefined,
downloadFsEntry: undefined,
uploadFilesAPI: undefined,
uploadMeta: undefined,
@@ -946,17 +962,61 @@ export const IOActionsContextProvider = ({
.catch(() => {});
}
- function fetchUrl(url: string, targetPath: string, haveProgress: boolean) {
+ /**
+ * Native HTTP GET on Capacitor (bypasses the WKWebView CORS that makes
+ * cross-origin fetch() fail with "Load failed"). The native call lives in the
+ * Capacitor IO module; this is just the platform indirection.
+ */
+ function capacitorHttpGet(
+ url: string,
+ ): Promise<{ base64: string; contentType: string }> {
+ // eslint-disable-next-line global-require
+ const ioAPI = require('-/services/io-capacitor');
+ return ioAPI.httpGet(url);
+ }
+
+ function fetchUrl(
+ url: string,
+ targetPath: string,
+ haveProgress: boolean,
+ objectStore: boolean,
+ ) {
if (AppConfig.isElectron) {
- return window.electronIO.ipcRenderer.invoke(
- 'fetchUrl',
- url,
- targetPath,
- haveProgress,
- );
+ // Object-store targets need the bytes back in the renderer so they can be
+ // uploaded into the bucket; local targets are streamed straight to disk in
+ // the main process. Both requests go through the Chromium network stack
+ // (browser User-Agent, no renderer CSP / server CORS limits).
+ if (objectStore) {
+ return window.electronIO.ipcRenderer
+ .invoke('fetchUrlBuffer', url)
+ .then((res) => {
+ if (res?.error) {
+ throw new Error(res.error);
+ }
+ return { arrayBuffer: () => Promise.resolve(res.data) };
+ });
+ }
+ return window.electronIO.ipcRenderer
+ .invoke('fetchUrl', url, targetPath, haveProgress)
+ .then((res) => {
+ if (res?.error) {
+ throw new Error(res.error);
+ }
+ return res;
+ });
+ }
+ if (AppConfig.isCapacitor) {
+ // WKWebView rejects cross-origin fetch() from the capacitor:// origin
+ // ("Load failed"). Route through the native HTTP plugin, which isn't bound
+ // by CORS, and expose an arrayBuffer() so saveFile() takes the same path
+ // as the web/object-store branch.
+ return capacitorHttpGet(url).then(({ base64 }) => ({
+ arrayBuffer: () => Promise.resolve(base64ToUint8Array(base64)),
+ }));
}
return fetch(url);
}
+
/**
* @param url
* @param targetPath
@@ -985,7 +1045,12 @@ export const IOActionsContextProvider = ({
return saveFilePromise({ path: targetPath }, arrayBuffer, true);
});
}
- return fetchUrl(url, targetPath, onDownloadProgress !== undefined)
+ return fetchUrl(
+ url,
+ targetPath,
+ onDownloadProgress !== undefined,
+ location.haveObjectStoreSupport(),
+ )
.then((response) => saveFile(response))
.then((fsEntry: TS.FileSystemEntry) => {
return generateThumbnailPromise(
@@ -1020,6 +1085,111 @@ export const IOActionsContextProvider = ({
});
}
+ /** Fetch a page's HTML, bypassing CORS/CSP via the main process on Electron. */
+ function fetchPageHtml(url: string): Promise<string> {
+ if (AppConfig.isElectron) {
+ return window.electronIO.ipcRenderer
+ .invoke('fetchUrlBuffer', url)
+ .then((res) => {
+ if (res?.error) {
+ throw new Error(res.error);
+ }
+ return new TextDecoder('utf-8').decode(res.data);
+ });
+ }
+ if (AppConfig.isCapacitor) {
+ return capacitorHttpGet(url).then(({ base64 }) =>
+ new TextDecoder('utf-8').decode(base64ToUint8Array(base64)),
+ );
+ }
+ return fetch(url).then((r) => {
+ if (!r.ok) {
+ throw new Error(`${r.status} ${r.statusText}`);
+ }
+ return r.text();
+ });
+ }
+
+ function blobToDataUrl(blob: Blob): Promise<string> {
+ return new Promise((resolve) => {
+ const reader = new FileReader();
+ reader.onloadend = () => resolve(reader.result as string);
+ reader.readAsDataURL(blob);
+ });
+ }
+
+ /**
+ * Resolve an image URL to a base64 data URL (CORS/CSP-free on Electron).
+ * Image URLs come from untrusted page content, so fetch them without the app
+ * session's cookies and with size/time caps to limit SSRF/DoS exposure.
+ */
+ function fetchImageDataUrl(url: string): Promise<string | null> {
+ if (AppConfig.isElectron) {
+ return window.electronIO.ipcRenderer
+ .invoke('fetchUrlBuffer', url, {
+ omitCredentials: true,
+ maxBytes: 15 * 1024 * 1024,
+ timeoutMs: 30000,
+ })
+ .then((res) => {
+ if (res?.error || !res?.data) {
+ return null;
+ }
+ const blob = new Blob([res.data], {
+ type: res.contentType || 'application/octet-stream',
+ });
+ return blobToDataUrl(blob);
+ })
+ .catch(() => null);
+ }
+ if (AppConfig.isCapacitor) {
+ // Build the data URL straight from the native base64 body.
+ return capacitorHttpGet(url)
+ .then(({ base64, contentType }) =>
+ base64 ? `data:${contentType || 'image/*'};base64,${base64}` : null,
+ )
+ .catch(() => null);
+ }
+ return fetch(url, { credentials: 'omit' })
+ .then((r) => r.blob())
+ .then(blobToDataUrl)
+ .catch(() => null);
+ }
+
+ /**
+ * Download an HTML page and save it as cleaned HTML or Markdown (scripts
+ * stripped, images optionally inlined as data URLs, optional article
+ * extraction). Binary URLs should use downloadUrl instead.
+ */
+ function downloadUrlAs(
+ url: string,
+ targetPath: string,
+ format: 'html' | 'markdown',
+ options: {
+ extractArticle?: boolean;
+ embedImages?: boolean;
+ tags?: string;
+ } = {},
+ ): Promise<TS.FileSystemEntry> {
+ const opts: ClipOptions = {
+ sourceUrl: url,
+ fetchImage: fetchImageDataUrl,
+ extractArticle: options.extractArticle,
+ embedImages: options.embedImages,
+ tags: options.tags,
+ scrappedOn: new Date().toISOString(),
+ };
+ return fetchPageHtml(url)
+ .then((html) =>
+ format === 'markdown'
+ ? htmlToMarkdown(html, opts)
+ : htmlToCleanHtml(html, opts),
+ )
+ .then((content) =>
+ saveTextFilePromise({ path: targetPath }, content, true),
+ );
+ }
+
function downloadFsEntry(fsEntry: TS.FileSystemEntry) {
const loc = findLocation(fsEntry.locationID);
if (!loc) {
@@ -2689,6 +2859,7 @@ export const IOActionsContextProvider = ({
copyDirs,
copyFiles,
downloadUrl,
+ downloadUrlAs,
downloadFsEntry,
uploadFilesAPI,
uploadFiles,
diff --git a/src/renderer/locales/en/core.json b/src/renderer/locales/en/core.json
index 926cf9cc9..7b96a4d77 100644
--- a/src/renderer/locales/en/core.json
+++ b/src/renderer/locales/en/core.json
@@ -796,6 +796,11 @@
"resetFileType": "Reset File Types",
"override": "Override",
"saveAs": "Save As",
+ "originalFile": "Original file",
+ "cleanedHtml": "Cleaned HTML",
+ "markdown": "Markdown",
+ "extractMainArticle": "Extract main article",
+ "embedImages": "Embed images",
"resolveConflictTitle": "File was modified externally",
"resolveConflictDesc": "For saving your changes as new file click on 'Save As' button, for override the external changes click the 'Override' button.",
"create": "Create",
diff --git a/src/renderer/perspectives/common/NewSubMenu.tsx b/src/renderer/perspectives/common/NewSubMenu.tsx
index c9507f519..f44a1d095 100644
--- a/src/renderer/perspectives/common/NewSubMenu.tsx
+++ b/src/renderer/perspectives/common/NewSubMenu.tsx
@@ -19,6 +19,7 @@ import AppConfig from '-/AppConfig';
import {
AddExistingFileIcon,
AudioRecordIcon,
+ DownloadIcon,
HTMLFileIcon,
LinkFileIcon,
MarkdownFileIcon,
@@ -27,7 +28,8 @@ import {
SmallArrowRightIcon,
TemplateFileIcon,
} from '-/components/CommonIcons';
-import { ProLabel } from '-/components/HelperComponents';
+import { useDownloadUrlDialogContext } from '-/components/dialogs/hooks/useDownloadUrlDialogContext';
+import { BetaLabel, ProLabel } from '-/components/HelperComponents';
import InfoIcon from '-/components/InfoIcon';
import TsMenuList from '-/components/TsMenuList';
import { Pro } from '-/pro';
@@ -67,6 +69,7 @@ function NewSubMenu(props: Props) {
addExistingFile,
} = props;
const hideProFeatures: boolean = useSelector(isHideProFeatures);
+ const { openDownloadUrl } = useDownloadUrlDialogContext();
const [anchorEl, setAnchorEl] = useState<HTMLElement | null>(null);
// Run a create action and dismiss the whole menu chain.
@@ -222,6 +225,23 @@ function NewSubMenu(props: Props) {
<ListItemText primary={t('core:addFiles')} />
</MenuItem>
)}
+ <MenuItem
+ key="newFromDownloadURL"
+ data-tid="newFromDownloadURLTID"
+ onClick={() => run(() => openDownloadUrl())}
+ >
+ <ListItemIcon>
+ <DownloadIcon />
+ </ListItemIcon>
+ <ListItemText
+ primary={
+ <>
+ {t('core:newFromDownloadURL')}
+ <BetaLabel />
+ </>
+ }
+ />
+ </MenuItem>
</TsMenuList>
</Menu>
</>
diff --git a/src/renderer/services/io-capacitor.ts b/src/renderer/services/io-capacitor.ts
index dfbdabfba..d7082400f 100644
--- a/src/renderer/services/io-capacitor.ts
+++ b/src/renderer/services/io-capacitor.ts
@@ -20,7 +20,7 @@ const {
const U = require('./capacitor-io-utils');
// Capacitor imports
-const { Capacitor } = require('@capacitor/core');
+const { Capacitor, CapacitorHttp } = require('@capacitor/core');
const { Filesystem, Directory, Encoding } = require('@capacitor/filesystem');
const { App } = require('@capacitor/app');
const { Device } = require('@capacitor/device');
@@ -1143,6 +1143,32 @@ function shareFiles(files) {
});
}
+/**
+ * Fetch a remote URL over the native HTTP stack. WKWebView rejects cross-origin
+ * fetch() from the capacitor:// origin with "Load failed"; CapacitorHttp runs
+ * the request natively and isn't bound by CORS. Returns the body as a base64
+ * string plus the content-type (the shape the native plugin yields for a binary
+ * responseType), leaving byte/data-URL conversion to the caller.
+ */
+function httpGet(url) {
+ return CapacitorHttp.request({
+ url,
+ method: 'GET',
+ responseType: 'arraybuffer',
+ }).then((res) => {
+ if (res.status < 200 || res.status >= 300) {
+ throw new Error('HTTP ' + res.status);
+ }
+ const contentType =
+ (res.headers &&
+ (res.headers['content-type'] || res.headers['Content-Type'])) ||
+ '';
+ // Native returns the binary body as a base64 string.
+ const base64 = typeof res.data === 'string' ? res.data : '';
+ return { base64, contentType };
+ });
+}
+
/**
* Download a file under Capacitor. Replaces the Cordova-only
* `window.plugins.Downloader`, which is undefined under Capacitor.
@@ -1278,5 +1304,6 @@ export {
focusWindow,
shareFiles,
downloadFile,
+ httpGet,
getNativeFileUrlAsync,
};
diff --git a/src/renderer/services/web-content.ts b/src/renderer/services/web-content.ts
new file mode 100644
index 000000000..aecc1d52f
--- /dev/null
+++ b/src/renderer/services/web-content.ts
@@ -0,0 +1,381 @@
+/**
+ * TagSpaces - universal file and folder organizer
+ * Copyright (C) 2017-present TagSpaces GmbH
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License (version 3) as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+/**
+ * Platform-agnostic web-content processing ported from the TagSpaces browser
+ * extension (../browser-ext: src/ui/popup.js + src/lib/utils.js). It turns a raw
+ * HTML page into either sanitized HTML or Markdown, optionally extracting the
+ * main article and inlining images as data URLs.
+ *
+ * All IO is injected (`fetchImage`) so this module stays free of `window`,
+ * `electron` or `chrome` references.
+ */
+
+import { isProbablyReaderable, Readability } from '@mozilla/readability';
+import DOMPurify from 'dompurify';
+import TurndownService from 'turndown';
+
+export interface ClipOptions {
+ /** Absolute URL of the page, used to resolve relative image/asset URLs. */
+ sourceUrl: string;
+ /** Returns a `data:` URL for the given absolute image URL, or null on failure. */
+ fetchImage: (url: string) => Promise<string | null>;
+ /** Run Readability to keep only the main article. */
+ extractArticle?: boolean;
+ /** Inline referenced images as base64 `data:` URLs. */
+ embedImages?: boolean;
+ /** Comma-separated tags written into the Markdown front matter. */
+ tags?: string;
+ /** ISO timestamp stamped into metadata (injected so the package stays pure). */
+ scrappedOn?: string;
+}
+
+type ImageTask = () => Promise<void>;
+
+const IMG_BATCH_SIZE = 5;
+// Cap how many images we inline so a hostile page can't trigger thousands of
+// sub-resource fetches / a giant output file.
+const MAX_INLINE_IMAGES = 200;
+
+// Only inline images fetched over http/https. Image URLs come from untrusted
+// page content, so block file:/data:/intranet schemes from being fetched.
+function isHttpUrl(url: string): boolean {
+ return /^https?:\/\//i.test(url);
+}
+
+// ---------------------------------------------------------------------------
+// Front matter helpers (ported from popup.js:914-956)
+// ---------------------------------------------------------------------------
+
+export function escapeYamlString(str: string): string {
+ return str.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
+}
+
+function formatFrontMatterEntry(key: string, value: string): string {
+ if (key === 'tags') {
+ const tags = value
+ .split(',')
+ .map((t) => t.trim())
+ .filter(Boolean)
+ .map((tag) => ` - "${escapeYamlString(tag)}"\n`)
+ .join('');
+ return `tags:\n${tags}`;
+ }
+ return `${key}: "${escapeYamlString(value)}"\n`;
+}
+
+/** Inject or update a YAML front-matter block at the top of the content. */
+export function injectFrontMatter(
+ content: string,
+ newMeta: Record<string, string>,
+): string {
+ const frontMatterRegex = /^---\n([\s\S]*?)\n---/;
+ const match = content.match(frontMatterRegex);
+
+ if (match) {
+ const existingYaml = match[1];
+ const additionalYaml = Object.entries(newMeta)
+ .filter(([key]) => !existingYaml.includes(`${key}:`))
+ .map(([key, value]) => formatFrontMatterEntry(key, value))
+ .join('');
+ return content.replace(
+ frontMatterRegex,
+ `---\n${existingYaml}\n${additionalYaml}---`,
+ );
+ }
+
+ const yamlBlock = Object.entries(newMeta)
+ .map(([key, value]) => formatFrontMatterEntry(key, value))
+ .join('');
+ return `---\n${yamlBlock}---\n\n${content}`;
+}
+
+// ---------------------------------------------------------------------------
+// Image helpers (ported from utils.js:20-61 + popup.js batching)
+// ---------------------------------------------------------------------------
+
+/** Robustly parse srcset and pick the highest-resolution URL (utils.js:20). */
+export function getHighestResUrl(el: Element, baseUrl: string): string | null {
+ const src = el.getAttribute('src');
+ const srcset = el.getAttribute('srcset');
+ const candidates: { url: string; value: number; type: 'w' | 'x' }[] = [];
+
+ if (src) {
+ candidates.push({ url: src, value: 1, type: 'x' });
+ }
+
+ if (srcset) {
+ const regex = /([^,\s]+)\s*(?:(\d+)w|(\d+(?:\.\d+)?)x)?/g;
+ Array.from(srcset.matchAll(regex)).forEach((match) => {
+ const url = match[1];
+ if (!url) {
+ return;
+ }
+ const wDesc = match[2] ? parseInt(match[2], 10) : null;
+ const xDesc = match[3] ? parseFloat(match[3]) : null;
+ if (wDesc) {
+ candidates.push({ url, value: wDesc, type: 'w' });
+ } else if (xDesc) {
+ candidates.push({ url, value: xDesc, type: 'x' });
+ } else {
+ candidates.push({ url, value: 1, type: 'x' });
+ }
+ });
+ }
+
+ if (candidates.length === 0) {
+ return null;
+ }
+
+ // Rank candidates: prefer width ('w') descriptors over density ('x').
+ const best = candidates.reduce((prev, curr) => {
+ if (curr.type === 'w' && prev.type === 'w') {
+ return curr.value > prev.value ? curr : prev;
+ }
+ if (curr.type === 'w' && prev.type === 'x') {
+ return curr;
+ }
+ if (curr.type === 'x' && prev.type === 'w') {
+ return prev;
+ }
+ return curr.value > prev.value ? curr : prev;
+ });
+
+ try {
+ return new URL(best.url, baseUrl).href;
+ } catch (e) {
+ return null;
+ }
+}
+
+async function processInBatches(
+ tasks: ImageTask[],
+ batchSize: number,
+): Promise<void> {
+ for (let i = 0; i < tasks.length; i += batchSize) {
+ const batch = tasks.slice(i, i + batchSize);
+ // eslint-disable-next-line no-await-in-loop
+ await Promise.all(batch.map((task) => task()));
+ }
+}
+
+/** Inline `<img>`/`<video poster>` sources of a document as data URLs in place. */
+async function inlineImagesInDoc(
+ doc: Document,
+ opts: ClipOptions,
+): Promise<void> {
+ const targets: { selector: string; attribute: string }[] = [
+ { selector: 'img', attribute: 'src' },
+ { selector: 'video[poster]', attribute: 'poster' },
+ ];
+
+ const tasks: ImageTask[] = targets.flatMap((target) =>
+ Array.from(doc.querySelectorAll(target.selector))
+ .map((el): ImageTask | null => {
+ const originalUrl = el.getAttribute(target.attribute);
+ if (!originalUrl || originalUrl.startsWith('data:')) {
+ return null;
+ }
+ let absoluteUrl: string | null;
+ try {
+ absoluteUrl =
+ target.selector === 'img'
+ ? getHighestResUrl(el, opts.sourceUrl)
+ : new URL(originalUrl, opts.sourceUrl).href;
+ } catch (e) {
+ return null;
+ }
+ if (!absoluteUrl || !isHttpUrl(absoluteUrl)) {
+ return null;
+ }
+ const url = absoluteUrl;
+ return async () => {
+ const dataUrl = await opts.fetchImage(url);
+ if (dataUrl && dataUrl.startsWith('data:image')) {
+ el.setAttribute(target.attribute, dataUrl);
+ el.removeAttribute('srcset');
+ }
+ };
+ })
+ .filter((task): task is ImageTask => task !== null),
+ );
+
+ await processInBatches(tasks.slice(0, MAX_INLINE_IMAGES), IMG_BATCH_SIZE);
+}
+
+/** Inline images referenced inside Markdown as data URLs (popup.js:824). */
+async function inlineImagesInMarkdown(
+ markdown: string,
+ opts: ClipOptions,
+): Promise<string> {
+ const regexTargets = [
+ { regex: /!\[(.*?)\]\(((?!data:).+?)(?:\s+"(.*?)")?\)/g, urlIndex: 2 },
+ { regex: /<img\s+[^>]*src=["']((?!data:)[^"']+)["'][^>]*>/g, urlIndex: 1 },
+ ];
+ const replacements = new Map<string, string | null>();
+ const tasks: ImageTask[] = [];
+
+ regexTargets.forEach((target) => {
+ Array.from(markdown.matchAll(target.regex)).forEach((match) => {
+ const originalUrl = match[target.urlIndex];
+ if (!originalUrl || replacements.has(originalUrl)) {
+ return;
+ }
+ let absoluteUrl: string;
+ try {
+ absoluteUrl = new URL(originalUrl, opts.sourceUrl).href;
+ } catch (e) {
+ return;
+ }
+ if (!isHttpUrl(absoluteUrl)) {
+ return;
+ }
+ replacements.set(originalUrl, null);
+ tasks.push(async () => {
+ const dataUrl = await opts.fetchImage(absoluteUrl);
+ if (dataUrl && dataUrl.startsWith('data:image')) {
+ replacements.set(originalUrl, dataUrl);
+ } else {
+ replacements.delete(originalUrl);
+ }
+ });
+ });
+ });
+
+ await processInBatches(tasks.slice(0, MAX_INLINE_IMAGES), IMG_BATCH_SIZE);
+
+ let result = markdown;
+ Array.from(replacements.entries()).forEach(([originalUrl, dataUrl]) => {
+ if (dataUrl) {
+ const escapedUrl = originalUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ result = result.replace(new RegExp(escapedUrl, 'g'), dataUrl);
+ }
+ });
+ return result;
+}
+
+// ---------------------------------------------------------------------------
+// Core sanitize / extract pipeline
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse raw HTML, optionally reduce it to the Readability article, and sanitize
+ * it (DOMPurify strips `<script>` and inline event handlers). Returns a fresh
+ * sanitized Document.
+ */
+function buildCleanDoc(html: string, opts: ClipOptions): Document {
+ const parser = new DOMParser();
+ let workingHtml = html;
+
+ if (opts.extractArticle) {
+ try {
+ const doc = parser.parseFromString(html, 'text/html');
+ if (isProbablyReaderable(doc)) {
+ const article = new Readability(doc).parse();
+ if (article && article.content) {
+ const title = article.title
+ ? `<h1>${article.title
+ .replace(/&/g, '&amp;')
+ .replace(/</g, '&lt;')
+ .replace(/>/g, '&gt;')}</h1>\n`
+ : '';
+ workingHtml = title + article.content;
+ }
+ }
+ } catch (e) {
+ // fall back to the full page on any Readability failure
+ }
+ }
+
+ const clean = DOMPurify.sanitize(workingHtml, { WHOLE_DOCUMENT: true });
+ return parser.parseFromString(clean, 'text/html');
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/** Produce sanitized, self-contained HTML with embedded source metadata. */
+export async function htmlToCleanHtml(
+ html: string,
+ opts: ClipOptions,
+): Promise<string> {
+ const doc = buildCleanDoc(html, opts);
+
+ if (opts.embedImages !== false) {
+ await inlineImagesInDoc(doc, opts);
+ }
+
+ let { head } = doc;
+ if (!head) {
+ head = doc.createElement('head');
+ doc.documentElement.insertBefore(head, doc.body);
+ }
+ if (!head.querySelector('meta[charset]')) {
+ const charsetMeta = doc.createElement('meta');
+ charsetMeta.setAttribute('charset', 'utf-8');
+ head.insertBefore(charsetMeta, head.firstChild);
+ }
+
+ const { body } = doc;
+ if (body) {
+ body.setAttribute('data-createdwith', 'TagSpaces');
+ body.setAttribute('data-sourceurl', opts.sourceUrl);
+ if (opts.scrappedOn) {
+ body.setAttribute('data-scrappedon', opts.scrappedOn);
+ }
+ }
+
+ const doctype = doc.doctype
+ ? `${new XMLSerializer().serializeToString(doc.doctype)}\n`
+ : '<!DOCTYPE html>\n';
+ return doctype + doc.documentElement.outerHTML;
+}
+
+/** Convert the (optionally article-extracted) page to Markdown with front matter. */
+export async function htmlToMarkdown(
+ html: string,
+ opts: ClipOptions,
+): Promise<string> {
+ const doc = buildCleanDoc(html, opts);
+ const turndownService = new TurndownService({
+ headingStyle: 'atx',
+ codeBlockStyle: 'fenced',
+ fence: '```',
+ });
+
+ let markdown = '';
+ try {
+ markdown = turndownService.turndown(doc.body ? doc.body.innerHTML : '');
+ } catch (e) {
+ markdown = '';
+ }
+
+ if (opts.embedImages !== false) {
+ markdown = await inlineImagesInMarkdown(markdown, opts);
+ }
+
+ const metadata: Record<string, string> = {
+ url: opts.sourceUrl,
+ date: opts.scrappedOn || '',
+ };
+ if (opts.tags && opts.tags.trim()) {
+ metadata.tags = opts.tags.trim();
+ }
+ return injectFrontMatter(markdown, metadata);
+}
diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.js
new file mode 100644
index 000000000..09a871624
--- /dev/null
+++ b/tests/e2e/download-url.pw.e2e.js
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2016-present - TagSpaces GmbH. All rights reserved.
+ */
+import fs from 'fs';
+import http from 'http';
+import path from 'path';
+
+import { expect, test } from './fixtures';
+import {
+ clickOn,
+ clickOnIfVisible,
+ expectElementExist,
+ getGridFileSelector,
+ reloadDirectory,
+ setInputValue,
+} from './general.helpers';
+import { startTestingApp, stopApp } from './hook';
+import {
+ createPwLocation,
+ createS3Location,
+ defaultLocationName,
+} from './location.helpers';
+import { clickOnMenuOperation } from './test-utils';
+import { clearDataStorage, closeWelcomePlaywright } from './welcome.helpers';
+
+// A tiny localhost HTTP server stands in for an arbitrary remote site so the
+// download path is exercised end-to-end without network flakiness. The Electron
+// main process reaches it over net.fetch the same way it would a real URL.
+const SAMPLE_TXT = 'TagSpaces download URL test content';
+const PAGE_HTML = `<!DOCTYPE html>
+<html>
+ <head><title>Clip Test Page</title></head>
+ <body>
+ <h1>Hello Heading</h1>
+ <p>Some paragraph text for clipping.</p>
+ <script>window.__evil = 'should-be-stripped';</script>
+ </body>
+</html>`;
+
+let server;
+let baseUrl;
+
+async function openDownloadUrlDialog() {
+ await clickOn('[data-tid=folderContainerOpenDirMenu]');
+ await clickOnMenuOperation('newFromDownloadURLTID');
+ await global.client.waitForSelector('[data-tid=downloadUrlDialogTID]', {
+ state: 'visible',
+ timeout: 8000,
+ });
+}
+
+test.beforeAll(async ({ isWeb, isS3, webServerPort }, testInfo) => {
+ server = http.createServer((req, res) => {
+ if (req.url === '/sample.txt') {
+ res.writeHead(200, { 'Content-Type': 'text/plain' });
+ res.end(SAMPLE_TXT);
+ } else if (req.url === '/page.html') {
+ res.writeHead(200, { 'Content-Type': 'text/html' });
+ res.end(PAGE_HTML);
+ } else {
+ res.writeHead(404);
+ res.end('not found');
+ }
+ });
+ await new Promise((resolve) => {
+ server.listen(0, '127.0.0.1', resolve);
+ });
+ baseUrl = `http://127.0.0.1:${server.address().port}`;
+
+ if (isS3) {
+ await startTestingApp({ isWeb, isS3, webServerPort, testInfo });
+ await closeWelcomePlaywright();
+ } else {
+ await startTestingApp(
+ { isWeb, isS3, webServerPort, testInfo },
+ 'extconfig.js',
+ );
+ }
+});
+
+test.afterAll(async () => {
+ await stopApp();
+ if (server) {
+ await new Promise((resolve) => server.close(resolve));
+ }
+});
+
+test.afterEach(async ({ isS3, testDataDir }) => {
+ // Keep the shared local test data clean (S3 is reset by other suites' refresh).
+ if (!isS3) {
+ ['sample.txt', 'page.md', 'page.html'].forEach((name) => {
+ const p = path.join(testDataDir, name);
+ if (fs.existsSync(p)) {
+ fs.rmSync(p, { force: true });
+ }
+ });
+ }
+ await clearDataStorage();
+});
+
+test.beforeEach(async ({ isS3, testDataDir }) => {
+ if (isS3) {
+ await createS3Location('', defaultLocationName, true);
+ await closeWelcomePlaywright();
+ } else {
+ await createPwLocation(testDataDir, defaultLocationName, true);
+ }
+ await clickOn('[data-tid=location_' + defaultLocationName + ']');
+ await expectElementExist(getGridFileSelector('empty_folder'), true, 15000);
+});
+
+test.describe('TST54 - Download from URL', () => {
+ test('TST5420 - Download raw file into the location [electron,s3]', async () => {
+ await openDownloadUrlDialog();
+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/sample.txt');
+ // Default "Original file" format → straight binary download into the location.
+ await clickOn('[data-tid=downloadFileUrlTID]');
+ await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);
+ // The raw download opens the upload-progress dialog — dismiss it before
+ // interacting with the directory menu again.
+ await clickOnIfVisible('[data-tid=closeFileUploadTID]');
+ await reloadDirectory();
+ await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);
+ });
+
+ test('TST5421 - Clip an HTML page as Markdown [electron,s3]', async ({
+ isS3,
+ testDataDir,
+ }) => {
+ await openDownloadUrlDialog();
+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');
+ await clickOn('[data-tid=downloadFormatMarkdownTID]');
+ await clickOn('[data-tid=downloadFileUrlTID]');
+ // page.html → page.md saved into the location.
+ await expectElementExist(getGridFileSelector('page.md'), true, 20000);
+
+ // On local locations assert the conversion result on disk: heading/text
+ // survive, the <script> is stripped by DOMPurify.
+ if (!isS3) {
+ const md = fs.readFileSync(path.join(testDataDir, 'page.md'), 'utf8');
+ expect(md).toContain('Hello Heading');
+ expect(md).toContain('Some paragraph text');
+ expect(md).not.toContain('should-be-stripped');
+ expect(md).not.toContain('<script');
+ }
+ });
+});
diff --git a/tests/e2e/test-utils.js b/tests/e2e/test-utils.js
index ab1b809bf..2858e6a02 100644
--- a/tests/e2e/test-utils.js
+++ b/tests/e2e/test-utils.js
@@ -23,6 +23,7 @@ const submenuParentTid = {
createNewAudioTID: 'createNewSubmenuTID',
newSubDirectory: 'createNewSubmenuTID',
addExistingFile: 'createNewSubmenuTID',
+ newFromDownloadURLTID: 'createNewSubmenuTID',
};
// Resolve the parent submenu trigger for a menu operation, if it lives in a