TagSpaces is an offline, open source, document manager with tagging support
chore: extending the downloadurl dialog
Ilian Sapundshiev4fb3fb8a3f02
diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsxindex aeb756df2..28f733682 100644--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx@@ -18,7 +18,9 @@import AppConfig from '-/AppConfig';import DraggablePaper from '-/components/DraggablePaper';+import TagsSelect from '-/components/TagsSelect';import TsButton from '-/components/TsButton';+import TsSwitch from '-/components/TsSwitch';import TsTextField from '-/components/TsTextField';import TsToggleButton from '-/components/TsToggleButton';import TargetPath from '-/components/dialogs/components/TargetPath';@@ -27,14 +29,22 @@ import TsDialogTitle from '-/components/dialogs/components/TsDialogTitle';import { TargetPathContextProvider } from '-/components/dialogs/hooks/TargetPathContextProvider';import { useFileUploadDialogContext } from '-/components/dialogs/hooks/useFileUploadDialogContext';import { useTargetPathContext } from '-/components/dialogs/hooks/useTargetPathContext';+import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';import { useIOActionsContext } from '-/hooks/useIOActionsContext';import { useNotificationContext } from '-/hooks/useNotificationContext';import { actions as AppActions, AppDispatch } from '-/reducers/app';+import {+ getFileNameTagPlace,+ getPrefixTagContainer,+ getTagColor,+ getTagDelimiter,+ getTagTextColor,+} from '-/reducers/settings';import { TS } from '-/tagspaces.namespace';import {Box,- Checkbox,+ FormControl,FormControlLabel,ToggleButtonGroup,Typography,@@ -44,16 +54,21 @@ import DialogContent from '@mui/material/DialogContent';import Paper from '@mui/material/Paper';import { useTheme } from '@mui/material/styles';import useMediaQuery from '@mui/material/useMediaQuery';+import { formatDateTime4Tag } from '@tagspaces/tagspaces-common/misc';+import { generateFileName } from '@tagspaces/tagspaces-common/paths';+import { getUuid } from '@tagspaces/tagspaces-common/utils-io';import { saveAs } from 'file-saver';-import React, { useRef, useState } from 'react';+import React, { useEffect, useRef, useState } from 'react';import { useTranslation } from 'react-i18next';-import { useDispatch } from 'react-redux';+import { useDispatch, useSelector } from 'react-redux';interface Props {open: boolean;onClose: (event?: Object, reason?: string) => void;}+type SaveFormat = 'original' | 'html' | 'markdown';+// Derive a safe, single-segment filename from a URL-derived name. Decoding the// raw name can reintroduce path separators / `..` (e.g. "%2e%2e%2f"), so keep// only the final segment and drop traversal/control characters to prevent@@ -72,133 +87,247 @@ function sanitizeDownloadFileName(name: string): string {return /^\.+$/.test(cleaned) ? '' : cleaned;}+// Split a sanitized filename into base name and extension (incl. leading dot).+function splitNameExt(name: string): { base: string; ext: string } {+ const dot = name.lastIndexOf('.');+ return dot > 0+ ? { base: name.slice(0, dot), ext: name.slice(dot) }+ : { base: name, ext: '' };+}++// Best-effort base name + original extension from a URL.+function deriveNameFromUrl(urlStr: string): { base: string; ext: string } {+ let fileName: string;+ let pathParts: string[];+ const url = new URL(urlStr);+ if (url.pathname) {+ const i = url.pathname.lastIndexOf('/');+ if (i > -1) {+ fileName = url.pathname.substring(i + 1);+ if (!fileName) {+ pathParts = url.pathname.split('/').filter(Boolean);+ }+ } else {+ fileName = url.pathname;+ }+ }+ if (!fileName) {+ fileName = `${+ url.hostname ++ (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')+ }.html`;+ } else if (fileName.indexOf('.') === -1) {+ fileName = `${url.hostname}-${fileName}.html`;+ }+ return splitNameExt(sanitizeDownloadFileName(fileName) || 'download.html');+}++// Extension implied by the chosen save format (clipped formats override the+// page's original extension).+function extForFormat(format: SaveFormat, originalExt: string): string {+ if (format === 'markdown') return '.md';+ if (format === 'html') return '.html';+ return originalExt || '.html';+}+function DownloadUrlDialog(props: Props) {const { open, onClose } = props;const { t } = useTranslation();const theme = useTheme();const smallScreen = useMediaQuery(theme.breakpoints.down('md'));const { setReflectActions } = useEditedEntryContext();+ const { currentLocation } = useCurrentLocationContext();const { downloadUrl, downloadUrlAs } = useIOActionsContext();const { showNotification } = useNotificationContext();const { openFileUploadDialog } = useFileUploadDialogContext();const dispatch: AppDispatch = useDispatch();+ const tagDelimiter = useSelector(getTagDelimiter);+ const prefixTagContainer = useSelector(getPrefixTagContainer);+ const filenameTagPlacedAtEnd = useSelector(getFileNameTagPlace);+ const defaultTagColor = useSelector(getTagColor);+ const defaultTagTextColor = useSelector(getTagTextColor);+// Cleaned-HTML / Markdown conversion needs CORS/CSP-free fetching of the page// and its images — only the desktop and native-mobile builds can do that.const canConvert = AppConfig.isElectron || AppConfig.isNativeMobile;const { targetDirectoryPath } = useTargetPathContext();- const fileUrl = useRef<string>();+ const [url, setUrl] = useState<string>('');+ const [fileName, setFileName] = useState<string>('');+ const [tags, setTags] = useState<TS.Tag[]>([]);+ const inputTags = useRef<TS.Tag[]>([]);+ // Original extension derived from the URL, used when format is "original".+ const originalExt = useRef<string>('.html');const [invalidURL, setInvalidURL] = useState<boolean>(false);- const [saveFormat, setSaveFormat] = useState<- 'original' | 'html' | 'markdown'- >('original');+ const [saveFormat, setSaveFormat] = useState<SaveFormat>('original');const [extractArticle, setExtractArticle] = useState<boolean>(false);const [embedImages, setEmbedImages] = useState<boolean>(true);- const onUploadProgress = (progress, abort, fileName) => {- dispatch(AppActions.onUploadProgress(progress, abort, fileName));+ // Reset the form and seed an automatic datetime-stamp tag each time the+ // (kept-mounted) dialog opens.+ useEffect(() => {+ if (open) {+ setUrl('');+ setFileName('');+ originalExt.current = '.html';+ setSaveFormat('original');+ setExtractArticle(false);+ setEmbedImages(true);+ setInvalidURL(false);+ inputTags.current = [];+ setTags([+ {+ id: getUuid(),+ title: formatDateTime4Tag(new Date(), true),+ color: defaultTagColor,+ textcolor: defaultTagTextColor,+ },+ ]);+ }+ // Reset only on open/close; default tag colors are read at seed time.+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [open]);++ const onUploadProgress = (progress, abort, fileName2) => {+ dispatch(AppActions.onUploadProgress(progress, abort, fileName2));};const handleUrlChange = (event: React.ChangeEvent<HTMLInputElement>) => {- fileUrl.current = event.target.value;+ const { value } = event.target;+ setUrl(value);+ if (invalidURL) {+ setInvalidURL(false);+ }+ try {+ const { base, ext } = deriveNameFromUrl(value);+ originalExt.current = ext;+ setFileName(base + extForFormat(saveFormat, ext));+ } catch (e) {+ // incomplete/invalid URL — leave the filename as-is+ }};- function downloadURL() {- if (fileUrl.current) {- try {- const url = new URL(fileUrl.current);- if (invalidURL) {- setInvalidURL(false);- }- let fileName;- let pathParts;- if (url.pathname) {- const delimiterIndex = url.pathname.lastIndexOf('/');- if (delimiterIndex > -1) {- fileName = url.pathname.substring(delimiterIndex + 1);- if (!fileName) {- pathParts = url.pathname.split('/').filter(Boolean);- }- } else {- fileName = url.pathname;- }- }- if (!fileName) {- fileName = `${- url.hostname +- (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')- }.html`;- } else if (fileName.indexOf('.') === -1) {- fileName = `${url.hostname}-${fileName}.html`;- }+ const handleFormatChange = (_event, value: SaveFormat) => {+ if (!value) return;+ setSaveFormat(value);+ setFileName(+ (prev) =>+ splitNameExt(prev).base + extForFormat(value, originalExt.current),+ );+ };- const reflectOpen = (entry: TS.FileSystemEntry) => {- const reflectAction: TS.EditAction = {- action: 'add',- entry,- open: true,- source: 'upload',- };- setReflectActions(reflectAction);- };- const notifyError = (e: Error) => {- console.log('downloadFile error:', e);- dispatch(- AppActions.setProgress(fileUrl.current, -1, t('core:errorCORS')),- );- showNotification(- t('core:downloadFileError', { message: e.message }),- 'error',- true,- );- };+ function uniqueTags(tagsArray: TS.Tag[]): TS.Tag[] {+ return tagsArray.reduce((acc: TS.Tag[], current) => {+ if (!acc.some((item) => item.title === current.title)) {+ acc.push(current);+ }+ return acc;+ }, []);+ }- const safeFileName =- sanitizeDownloadFileName(fileName) || 'download.html';+ const handleTagsChange = (+ name: string,+ value: Array<TS.Tag>,+ action: string,+ ) => {+ if (action === 'remove-value') {+ const toRemove = value.map((tag) => tag.title);+ setTags(tags.filter((tag) => !toRemove.includes(tag.title)));+ } else {+ setTags(value);+ }+ };- if (canConvert && saveFormat !== 'original') {- // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline- // images and optionally extract the article, then save into the location.- const baseName = safeFileName.replace(/\.[^.]+$/, '') || 'download';- const ext = saveFormat === 'markdown' ? '.md' : '.html';- downloadUrlAs(- fileUrl.current,- `${targetDirectoryPath}/${baseName}${ext}`,- saveFormat,- { extractArticle, embedImages },- )- .then(reflectOpen)- .catch(notifyError);- } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {- // Raw download straight into the location (local + cloud + mobile),- // routed through the platform's CORS/CSP-free fetch.- dispatch(AppActions.resetProgress());- openFileUploadDialog();- downloadUrl(- fileUrl.current,- `${targetDirectoryPath}/${safeFileName}`,- onUploadProgress,- )- .then(reflectOpen)- .catch(notifyError);- } else {- // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain- // browser download.- saveAs(fileUrl.current, safeFileName);- }- onClose();- } catch (ex) {- setInvalidURL(true);- console.log('downloadURL', ex);+ const handleNewTags = (newTags: TS.Tag[]) => {+ if (newTags === undefined) {+ if (inputTags.current.length > 0) {+ setTags(uniqueTags([...tags, ...inputTags.current]));+ inputTags.current = [];}+ } else {+ inputTags.current = newTags;+ }+ };++ function downloadURL() {+ if (!url) {+ return;+ }+ try {+ // Validate the URL — throws for an incomplete entry.+ // eslint-disable-next-line no-new+ new URL(url);+ } catch (ex) {+ setInvalidURL(true);+ return;}++ // Embed the picked tags (plus any tag typed but not yet committed) into the+ // filename: name[tag1 tag2].ext+ const allTags = uniqueTags([...tags, ...inputTags.current]);+ const tagTitles = allTags.map((tag) => tag.title);+ const baseName =+ fileName || `download${extForFormat(saveFormat, originalExt.current)}`;+ const finalName = generateFileName(+ baseName,+ tagTitles,+ tagDelimiter,+ currentLocation?.getDirSeparator(),+ prefixTagContainer,+ filenameTagPlacedAtEnd,+ );+ const targetPath = `${targetDirectoryPath}/${finalName}`;++ const reflectOpen = (entry: TS.FileSystemEntry) => {+ setReflectActions({+ action: 'add',+ entry,+ open: true,+ source: 'upload',+ });+ };+ const notifyError = (e: Error) => {+ console.log('downloadFile error:', e);+ dispatch(AppActions.setProgress(url, -1, t('core:errorCORS')));+ showNotification(+ t('core:downloadFileError', { message: e.message }),+ 'error',+ true,+ );+ };++ if (canConvert && saveFormat !== 'original') {+ // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline+ // images and optionally extract the article, then save into the location.+ downloadUrlAs(url, targetPath, saveFormat, {+ extractArticle,+ embedImages,+ tags: tagTitles.join(', '),+ })+ .then(reflectOpen)+ .catch(notifyError);+ } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {+ // Raw download straight into the location (local + cloud + mobile),+ // routed through the platform's CORS/CSP-free fetch.+ dispatch(AppActions.resetProgress());+ openFileUploadDialog();+ downloadUrl(url, targetPath, onUploadProgress)+ .then(reflectOpen)+ .catch(notifyError);+ } else {+ // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain+ // browser download.+ saveAs(url, finalName);+ }+ onClose();}const okButton = (<TsButtonvariant="contained"data-tid="downloadFileUrlTID"+ disabled={!url}onClick={() => downloadURL()}sx={{@@ -229,7 +358,7 @@ function DownloadUrlDialog(props: Props) {/><DialogContentsx={{- minWidth: '200px',+ minWidth: '250px',marginBottom: '20px',overflow: 'overlay',}}@@ -240,6 +369,7 @@ function DownloadUrlDialog(props: Props) {label={t('core:url')}autoFocusname="name"+ value={url}data-tid="newUrlTID"onKeyDown={(event) => {if (event.key === 'Enter') {@@ -248,6 +378,33 @@ function DownloadUrlDialog(props: Props) {}}onChange={handleUrlChange}/>+ {url && (+ <FormControl fullWidth sx={{ mt: 2 }}>+ <TsTextField+ label={t('core:fileName')}+ name="downloadFileName"+ value={fileName}+ data-tid="downloadFileNameTID"+ onChange={(event) => setFileName(event.target.value)}+ onKeyDown={(event) => {+ if (event.key === 'Enter') {+ downloadURL();+ }+ }}+ />+ </FormControl>+ )}+ <Box sx={{ mt: 2 }}>+ <TagsSelect+ dataTid="downloadUrlTagsSelectTID"+ placeholderText={t('core:selectTags')}+ label={t('core:fileTags')}+ tags={tags}+ handleChange={handleTagsChange}+ handleNewTags={handleNewTags}+ tagMode="default"+ />+ </Box><TargetPath />{canConvert && (<Box sx={{ mt: 2 }}>@@ -262,9 +419,7 @@ function DownloadUrlDialog(props: Props) {size="small"value={saveFormat}exclusive- onChange={(_, v) => {- if (v) setSaveFormat(v);- }}+ onChange={handleFormatChange}data-tid="downloadFormatToggleTID"><TsToggleButton@@ -284,11 +439,12 @@ function DownloadUrlDialog(props: Props) {</TsToggleButton></ToggleButtonGroup>{saveFormat !== 'original' && (- <Box sx={{ mt: 1 }}>+ <Box sx={{ mt: 1, display: 'flex', flexDirection: 'column' }}><FormControlLabel+ labelPlacement="start"+ sx={{ ml: 0, justifyContent: 'space-between' }}control={- <Checkbox- size="small"+ <TsSwitchchecked={extractArticle}onChange={(e) => setExtractArticle(e.target.checked)}data-tid="downloadExtractArticleTID"@@ -297,9 +453,10 @@ function DownloadUrlDialog(props: Props) {label={t('core:extractMainArticle')}/><FormControlLabel+ labelPlacement="start"+ sx={{ ml: 0, justifyContent: 'space-between' }}control={- <Checkbox- size="small"+ <TsSwitchchecked={embedImages}onChange={(e) => setEmbedImages(e.target.checked)}data-tid="downloadEmbedImagesTID"diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.jsindex 09a871624..f38c00817 100644--- a/tests/e2e/download-url.pw.e2e.js+++ b/tests/e2e/download-url.pw.e2e.js@@ -40,6 +40,15 @@ const PAGE_HTML = `<!DOCTYPE html>let server;let baseUrl;+// Match a grid entry whose data-tid starts with the given base name, ignoring+// the auto datetime tag + extension that `dataTidFormat` turns into `_…`.+async function expectEntryStartingWith(base, timeout = 20000) {+ await global.client.waitForSelector(`[data-tid^="fsEntryName_${base}"]`, {+ state: 'visible',+ timeout,+ });+}+async function openDownloadUrlDialog() {await clickOn('[data-tid=folderContainerOpenDirMenu]');await clickOnMenuOperation('newFromDownloadURLTID');@@ -87,13 +96,12 @@ test.afterAll(async () => {test.afterEach(async ({ isS3, testDataDir }) => {// Keep the shared local test data clean (S3 is reset by other suites' refresh).+ // Downloads carry an auto datetime tag in the filename (e.g. sample[…].txt),+ // so match by prefix rather than exact name.if (!isS3) {- ['sample.txt', 'page.md', 'page.html'].forEach((name) => {- const p = path.join(testDataDir, name);- if (fs.existsSync(p)) {- fs.rmSync(p, { force: true });- }- });+ fs.readdirSync(testDataDir)+ .filter((name) => /^(sample|page|download)[[.]/.test(name))+ .forEach((name) => fs.rmSync(path.join(testDataDir, name), { force: true }));}await clearDataStorage();});@@ -113,14 +121,16 @@ test.describe('TST54 - Download from URL', () => {test('TST5420 - Download raw file into the location [electron,s3]', async () => {await openDownloadUrlDialog();await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/sample.txt');- // Default "Original file" format → straight binary download into the location.+ // Default "Original file" format → straight binary download into the+ // location. The filename carries an auto datetime tag (sample[…].txt), so+ // match the entry by prefix.await clickOn('[data-tid=downloadFileUrlTID]');- await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);+ await expectEntryStartingWith('sample');// The raw download opens the upload-progress dialog — dismiss it before// interacting with the directory menu again.await clickOnIfVisible('[data-tid=closeFileUploadTID]');await reloadDirectory();- await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);+ await expectEntryStartingWith('sample');});test('TST5421 - Clip an HTML page as Markdown [electron,s3]', async ({@@ -131,13 +141,17 @@ test.describe('TST54 - Download from URL', () => {await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');await clickOn('[data-tid=downloadFormatMarkdownTID]');await clickOn('[data-tid=downloadFileUrlTID]');- // page.html → page.md saved into the location.- await expectElementExist(getGridFileSelector('page.md'), true, 20000);+ // page.html → page[…datetime tag…].md saved into the location.+ await expectEntryStartingWith('page');// On local locations assert the conversion result on disk: heading/text// survive, the <script> is stripped by DOMPurify.if (!isS3) {- const md = fs.readFileSync(path.join(testDataDir, 'page.md'), 'utf8');+ const mdFile = fs+ .readdirSync(testDataDir)+ .find((name) => /^page.*\.md$/.test(name));+ expect(mdFile).toBeTruthy();+ const md = fs.readFileSync(path.join(testDataDir, mdFile), 'utf8');expect(md).toContain('Hello Heading');expect(md).toContain('Some paragraph text');expect(md).not.toContain('should-be-stripped');