Skip to content

TagSpaces is an offline, open source, document manager with tagging support

chore: extending the downloadurl dialog

Ilian Sapundshiev4fb3fb8a3f02
diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
index aeb756df2..28f733682 100644
--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx
+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
@@ -18,7 +18,9 @@
import AppConfig from '-/AppConfig';
import DraggablePaper from '-/components/DraggablePaper';
+import TagsSelect from '-/components/TagsSelect';
import TsButton from '-/components/TsButton';
+import TsSwitch from '-/components/TsSwitch';
import TsTextField from '-/components/TsTextField';
import TsToggleButton from '-/components/TsToggleButton';
import TargetPath from '-/components/dialogs/components/TargetPath';
@@ -27,14 +29,22 @@ import TsDialogTitle from '-/components/dialogs/components/TsDialogTitle';
import { TargetPathContextProvider } from '-/components/dialogs/hooks/TargetPathContextProvider';
import { useFileUploadDialogContext } from '-/components/dialogs/hooks/useFileUploadDialogContext';
import { useTargetPathContext } from '-/components/dialogs/hooks/useTargetPathContext';
+import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';
import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';
import { useIOActionsContext } from '-/hooks/useIOActionsContext';
import { useNotificationContext } from '-/hooks/useNotificationContext';
import { actions as AppActions, AppDispatch } from '-/reducers/app';
+import {
+ getFileNameTagPlace,
+ getPrefixTagContainer,
+ getTagColor,
+ getTagDelimiter,
+ getTagTextColor,
+} from '-/reducers/settings';
import { TS } from '-/tagspaces.namespace';
import {
Box,
- Checkbox,
+ FormControl,
FormControlLabel,
ToggleButtonGroup,
Typography,
@@ -44,16 +54,21 @@ import DialogContent from '@mui/material/DialogContent';
import Paper from '@mui/material/Paper';
import { useTheme } from '@mui/material/styles';
import useMediaQuery from '@mui/material/useMediaQuery';
+import { formatDateTime4Tag } from '@tagspaces/tagspaces-common/misc';
+import { generateFileName } from '@tagspaces/tagspaces-common/paths';
+import { getUuid } from '@tagspaces/tagspaces-common/utils-io';
import { saveAs } from 'file-saver';
-import React, { useRef, useState } from 'react';
+import React, { useEffect, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
-import { useDispatch } from 'react-redux';
+import { useDispatch, useSelector } from 'react-redux';
interface Props {
open: boolean;
onClose: (event?: Object, reason?: string) => void;
}
+type SaveFormat = 'original' | 'html' | 'markdown';
+
// Derive a safe, single-segment filename from a URL-derived name. Decoding the
// raw name can reintroduce path separators / `..` (e.g. "%2e%2e%2f"), so keep
// only the final segment and drop traversal/control characters to prevent
@@ -72,133 +87,247 @@ function sanitizeDownloadFileName(name: string): string {
return /^\.+$/.test(cleaned) ? '' : cleaned;
}
+// Split a sanitized filename into base name and extension (incl. leading dot).
+function splitNameExt(name: string): { base: string; ext: string } {
+ const dot = name.lastIndexOf('.');
+ return dot > 0
+ ? { base: name.slice(0, dot), ext: name.slice(dot) }
+ : { base: name, ext: '' };
+}
+
+// Best-effort base name + original extension from a URL.
+function deriveNameFromUrl(urlStr: string): { base: string; ext: string } {
+ let fileName: string;
+ let pathParts: string[];
+ const url = new URL(urlStr);
+ if (url.pathname) {
+ const i = url.pathname.lastIndexOf('/');
+ if (i > -1) {
+ fileName = url.pathname.substring(i + 1);
+ if (!fileName) {
+ pathParts = url.pathname.split('/').filter(Boolean);
+ }
+ } else {
+ fileName = url.pathname;
+ }
+ }
+ if (!fileName) {
+ fileName = `${
+ url.hostname +
+ (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')
+ }.html`;
+ } else if (fileName.indexOf('.') === -1) {
+ fileName = `${url.hostname}-${fileName}.html`;
+ }
+ return splitNameExt(sanitizeDownloadFileName(fileName) || 'download.html');
+}
+
+// Extension implied by the chosen save format (clipped formats override the
+// page's original extension).
+function extForFormat(format: SaveFormat, originalExt: string): string {
+ if (format === 'markdown') return '.md';
+ if (format === 'html') return '.html';
+ return originalExt || '.html';
+}
+
function DownloadUrlDialog(props: Props) {
const { open, onClose } = props;
const { t } = useTranslation();
const theme = useTheme();
const smallScreen = useMediaQuery(theme.breakpoints.down('md'));
const { setReflectActions } = useEditedEntryContext();
+ const { currentLocation } = useCurrentLocationContext();
const { downloadUrl, downloadUrlAs } = useIOActionsContext();
const { showNotification } = useNotificationContext();
const { openFileUploadDialog } = useFileUploadDialogContext();
const dispatch: AppDispatch = useDispatch();
+ const tagDelimiter = useSelector(getTagDelimiter);
+ const prefixTagContainer = useSelector(getPrefixTagContainer);
+ const filenameTagPlacedAtEnd = useSelector(getFileNameTagPlace);
+ const defaultTagColor = useSelector(getTagColor);
+ const defaultTagTextColor = useSelector(getTagTextColor);
+
// Cleaned-HTML / Markdown conversion needs CORS/CSP-free fetching of the page
// and its images — only the desktop and native-mobile builds can do that.
const canConvert = AppConfig.isElectron || AppConfig.isNativeMobile;
const { targetDirectoryPath } = useTargetPathContext();
- const fileUrl = useRef<string>();
+ const [url, setUrl] = useState<string>('');
+ const [fileName, setFileName] = useState<string>('');
+ const [tags, setTags] = useState<TS.Tag[]>([]);
+ const inputTags = useRef<TS.Tag[]>([]);
+ // Original extension derived from the URL, used when format is "original".
+ const originalExt = useRef<string>('.html');
const [invalidURL, setInvalidURL] = useState<boolean>(false);
- const [saveFormat, setSaveFormat] = useState<
- 'original' | 'html' | 'markdown'
- >('original');
+ const [saveFormat, setSaveFormat] = useState<SaveFormat>('original');
const [extractArticle, setExtractArticle] = useState<boolean>(false);
const [embedImages, setEmbedImages] = useState<boolean>(true);
- const onUploadProgress = (progress, abort, fileName) => {
- dispatch(AppActions.onUploadProgress(progress, abort, fileName));
+ // Reset the form and seed an automatic datetime-stamp tag each time the
+ // (kept-mounted) dialog opens.
+ useEffect(() => {
+ if (open) {
+ setUrl('');
+ setFileName('');
+ originalExt.current = '.html';
+ setSaveFormat('original');
+ setExtractArticle(false);
+ setEmbedImages(true);
+ setInvalidURL(false);
+ inputTags.current = [];
+ setTags([
+ {
+ id: getUuid(),
+ title: formatDateTime4Tag(new Date(), true),
+ color: defaultTagColor,
+ textcolor: defaultTagTextColor,
+ },
+ ]);
+ }
+ // Reset only on open/close; default tag colors are read at seed time.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [open]);
+
+ const onUploadProgress = (progress, abort, fileName2) => {
+ dispatch(AppActions.onUploadProgress(progress, abort, fileName2));
};
const handleUrlChange = (event: React.ChangeEvent<HTMLInputElement>) => {
- fileUrl.current = event.target.value;
+ const { value } = event.target;
+ setUrl(value);
+ if (invalidURL) {
+ setInvalidURL(false);
+ }
+ try {
+ const { base, ext } = deriveNameFromUrl(value);
+ originalExt.current = ext;
+ setFileName(base + extForFormat(saveFormat, ext));
+ } catch (e) {
+ // incomplete/invalid URL — leave the filename as-is
+ }
};
- function downloadURL() {
- if (fileUrl.current) {
- try {
- const url = new URL(fileUrl.current);
- if (invalidURL) {
- setInvalidURL(false);
- }
- let fileName;
- let pathParts;
- if (url.pathname) {
- const delimiterIndex = url.pathname.lastIndexOf('/');
- if (delimiterIndex > -1) {
- fileName = url.pathname.substring(delimiterIndex + 1);
- if (!fileName) {
- pathParts = url.pathname.split('/').filter(Boolean);
- }
- } else {
- fileName = url.pathname;
- }
- }
- if (!fileName) {
- fileName = `${
- url.hostname +
- (pathParts && pathParts.length > 0 ? pathParts.join('-') : '')
- }.html`;
- } else if (fileName.indexOf('.') === -1) {
- fileName = `${url.hostname}-${fileName}.html`;
- }
+ const handleFormatChange = (_event, value: SaveFormat) => {
+ if (!value) return;
+ setSaveFormat(value);
+ setFileName(
+ (prev) =>
+ splitNameExt(prev).base + extForFormat(value, originalExt.current),
+ );
+ };
- const reflectOpen = (entry: TS.FileSystemEntry) => {
- const reflectAction: TS.EditAction = {
- action: 'add',
- entry,
- open: true,
- source: 'upload',
- };
- setReflectActions(reflectAction);
- };
- const notifyError = (e: Error) => {
- console.log('downloadFile error:', e);
- dispatch(
- AppActions.setProgress(fileUrl.current, -1, t('core:errorCORS')),
- );
- showNotification(
- t('core:downloadFileError', { message: e.message }),
- 'error',
- true,
- );
- };
+ function uniqueTags(tagsArray: TS.Tag[]): TS.Tag[] {
+ return tagsArray.reduce((acc: TS.Tag[], current) => {
+ if (!acc.some((item) => item.title === current.title)) {
+ acc.push(current);
+ }
+ return acc;
+ }, []);
+ }
- const safeFileName =
- sanitizeDownloadFileName(fileName) || 'download.html';
+ const handleTagsChange = (
+ name: string,
+ value: Array<TS.Tag>,
+ action: string,
+ ) => {
+ if (action === 'remove-value') {
+ const toRemove = value.map((tag) => tag.title);
+ setTags(tags.filter((tag) => !toRemove.includes(tag.title)));
+ } else {
+ setTags(value);
+ }
+ };
- if (canConvert && saveFormat !== 'original') {
- // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline
- // images and optionally extract the article, then save into the location.
- const baseName = safeFileName.replace(/\.[^.]+$/, '') || 'download';
- const ext = saveFormat === 'markdown' ? '.md' : '.html';
- downloadUrlAs(
- fileUrl.current,
- `${targetDirectoryPath}/${baseName}${ext}`,
- saveFormat,
- { extractArticle, embedImages },
- )
- .then(reflectOpen)
- .catch(notifyError);
- } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {
- // Raw download straight into the location (local + cloud + mobile),
- // routed through the platform's CORS/CSP-free fetch.
- dispatch(AppActions.resetProgress());
- openFileUploadDialog();
- downloadUrl(
- fileUrl.current,
- `${targetDirectoryPath}/${safeFileName}`,
- onUploadProgress,
- )
- .then(reflectOpen)
- .catch(notifyError);
- } else {
- // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain
- // browser download.
- saveAs(fileUrl.current, safeFileName);
- }
- onClose();
- } catch (ex) {
- setInvalidURL(true);
- console.log('downloadURL', ex);
+ const handleNewTags = (newTags: TS.Tag[]) => {
+ if (newTags === undefined) {
+ if (inputTags.current.length > 0) {
+ setTags(uniqueTags([...tags, ...inputTags.current]));
+ inputTags.current = [];
}
+ } else {
+ inputTags.current = newTags;
+ }
+ };
+
+ function downloadURL() {
+ if (!url) {
+ return;
+ }
+ try {
+ // Validate the URL — throws for an incomplete entry.
+ // eslint-disable-next-line no-new
+ new URL(url);
+ } catch (ex) {
+ setInvalidURL(true);
+ return;
}
+
+ // Embed the picked tags (plus any tag typed but not yet committed) into the
+ // filename: name[tag1 tag2].ext
+ const allTags = uniqueTags([...tags, ...inputTags.current]);
+ const tagTitles = allTags.map((tag) => tag.title);
+ const baseName =
+ fileName || `download${extForFormat(saveFormat, originalExt.current)}`;
+ const finalName = generateFileName(
+ baseName,
+ tagTitles,
+ tagDelimiter,
+ currentLocation?.getDirSeparator(),
+ prefixTagContainer,
+ filenameTagPlacedAtEnd,
+ );
+ const targetPath = `${targetDirectoryPath}/${finalName}`;
+
+ const reflectOpen = (entry: TS.FileSystemEntry) => {
+ setReflectActions({
+ action: 'add',
+ entry,
+ open: true,
+ source: 'upload',
+ });
+ };
+ const notifyError = (e: Error) => {
+ console.log('downloadFile error:', e);
+ dispatch(AppActions.setProgress(url, -1, t('core:errorCORS')));
+ showNotification(
+ t('core:downloadFileError', { message: e.message }),
+ 'error',
+ true,
+ );
+ };
+
+ if (canConvert && saveFormat !== 'original') {
+ // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline
+ // images and optionally extract the article, then save into the location.
+ downloadUrlAs(url, targetPath, saveFormat, {
+ extractArticle,
+ embedImages,
+ tags: tagTitles.join(', '),
+ })
+ .then(reflectOpen)
+ .catch(notifyError);
+ } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {
+ // Raw download straight into the location (local + cloud + mobile),
+ // routed through the platform's CORS/CSP-free fetch.
+ dispatch(AppActions.resetProgress());
+ openFileUploadDialog();
+ downloadUrl(url, targetPath, onUploadProgress)
+ .then(reflectOpen)
+ .catch(notifyError);
+ } else {
+ // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain
+ // browser download.
+ saveAs(url, finalName);
+ }
+ onClose();
}
const okButton = (
<TsButton
variant="contained"
data-tid="downloadFileUrlTID"
+ disabled={!url}
onClick={() => downloadURL()}
sx={
{
@@ -229,7 +358,7 @@ function DownloadUrlDialog(props: Props) {
/>
<DialogContent
sx={{
- minWidth: '200px',
+ minWidth: '250px',
marginBottom: '20px',
overflow: 'overlay',
}}
@@ -240,6 +369,7 @@ function DownloadUrlDialog(props: Props) {
label={t('core:url')}
autoFocus
name="name"
+ value={url}
data-tid="newUrlTID"
onKeyDown={(event) => {
if (event.key === 'Enter') {
@@ -248,6 +378,33 @@ function DownloadUrlDialog(props: Props) {
}}
onChange={handleUrlChange}
/>
+ {url && (
+ <FormControl fullWidth sx={{ mt: 2 }}>
+ <TsTextField
+ label={t('core:fileName')}
+ name="downloadFileName"
+ value={fileName}
+ data-tid="downloadFileNameTID"
+ onChange={(event) => setFileName(event.target.value)}
+ onKeyDown={(event) => {
+ if (event.key === 'Enter') {
+ downloadURL();
+ }
+ }}
+ />
+ </FormControl>
+ )}
+ <Box sx={{ mt: 2 }}>
+ <TagsSelect
+ dataTid="downloadUrlTagsSelectTID"
+ placeholderText={t('core:selectTags')}
+ label={t('core:fileTags')}
+ tags={tags}
+ handleChange={handleTagsChange}
+ handleNewTags={handleNewTags}
+ tagMode="default"
+ />
+ </Box>
<TargetPath />
{canConvert && (
<Box sx={{ mt: 2 }}>
@@ -262,9 +419,7 @@ function DownloadUrlDialog(props: Props) {
size="small"
value={saveFormat}
exclusive
- onChange={(_, v) => {
- if (v) setSaveFormat(v);
- }}
+ onChange={handleFormatChange}
data-tid="downloadFormatToggleTID"
>
<TsToggleButton
@@ -284,11 +439,12 @@ function DownloadUrlDialog(props: Props) {
</TsToggleButton>
</ToggleButtonGroup>
{saveFormat !== 'original' && (
- <Box sx={{ mt: 1 }}>
+ <Box sx={{ mt: 1, display: 'flex', flexDirection: 'column' }}>
<FormControlLabel
+ labelPlacement="start"
+ sx={{ ml: 0, justifyContent: 'space-between' }}
control={
- <Checkbox
- size="small"
+ <TsSwitch
checked={extractArticle}
onChange={(e) => setExtractArticle(e.target.checked)}
data-tid="downloadExtractArticleTID"
@@ -297,9 +453,10 @@ function DownloadUrlDialog(props: Props) {
label={t('core:extractMainArticle')}
/>
<FormControlLabel
+ labelPlacement="start"
+ sx={{ ml: 0, justifyContent: 'space-between' }}
control={
- <Checkbox
- size="small"
+ <TsSwitch
checked={embedImages}
onChange={(e) => setEmbedImages(e.target.checked)}
data-tid="downloadEmbedImagesTID"
diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.js
index 09a871624..f38c00817 100644
--- a/tests/e2e/download-url.pw.e2e.js
+++ b/tests/e2e/download-url.pw.e2e.js
@@ -40,6 +40,15 @@ const PAGE_HTML = `<!DOCTYPE html>
let server;
let baseUrl;
+// Match a grid entry whose data-tid starts with the given base name, ignoring
+// the auto datetime tag + extension that `dataTidFormat` turns into `_…`.
+async function expectEntryStartingWith(base, timeout = 20000) {
+ await global.client.waitForSelector(`[data-tid^="fsEntryName_${base}"]`, {
+ state: 'visible',
+ timeout,
+ });
+}
+
async function openDownloadUrlDialog() {
await clickOn('[data-tid=folderContainerOpenDirMenu]');
await clickOnMenuOperation('newFromDownloadURLTID');
@@ -87,13 +96,12 @@ test.afterAll(async () => {
test.afterEach(async ({ isS3, testDataDir }) => {
// Keep the shared local test data clean (S3 is reset by other suites' refresh).
+ // Downloads carry an auto datetime tag in the filename (e.g. sample[…].txt),
+ // so match by prefix rather than exact name.
if (!isS3) {
- ['sample.txt', 'page.md', 'page.html'].forEach((name) => {
- const p = path.join(testDataDir, name);
- if (fs.existsSync(p)) {
- fs.rmSync(p, { force: true });
- }
- });
+ fs.readdirSync(testDataDir)
+ .filter((name) => /^(sample|page|download)[[.]/.test(name))
+ .forEach((name) => fs.rmSync(path.join(testDataDir, name), { force: true }));
}
await clearDataStorage();
});
@@ -113,14 +121,16 @@ test.describe('TST54 - Download from URL', () => {
test('TST5420 - Download raw file into the location [electron,s3]', async () => {
await openDownloadUrlDialog();
await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/sample.txt');
- // Default "Original file" format → straight binary download into the location.
+ // Default "Original file" format → straight binary download into the
+ // location. The filename carries an auto datetime tag (sample[…].txt), so
+ // match the entry by prefix.
await clickOn('[data-tid=downloadFileUrlTID]');
- await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);
+ await expectEntryStartingWith('sample');
// The raw download opens the upload-progress dialog — dismiss it before
// interacting with the directory menu again.
await clickOnIfVisible('[data-tid=closeFileUploadTID]');
await reloadDirectory();
- await expectElementExist(getGridFileSelector('sample.txt'), true, 20000);
+ await expectEntryStartingWith('sample');
});
test('TST5421 - Clip an HTML page as Markdown [electron,s3]', async ({
@@ -131,13 +141,17 @@ test.describe('TST54 - Download from URL', () => {
await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');
await clickOn('[data-tid=downloadFormatMarkdownTID]');
await clickOn('[data-tid=downloadFileUrlTID]');
- // page.html → page.md saved into the location.
- await expectElementExist(getGridFileSelector('page.md'), true, 20000);
+ // page.html → page[…datetime tag…].md saved into the location.
+ await expectEntryStartingWith('page');
// On local locations assert the conversion result on disk: heading/text
// survive, the <script> is stripped by DOMPurify.
if (!isS3) {
- const md = fs.readFileSync(path.join(testDataDir, 'page.md'), 'utf8');
+ const mdFile = fs
+ .readdirSync(testDataDir)
+ .find((name) => /^page.*\.md$/.test(name));
+ expect(mdFile).toBeTruthy();
+ const md = fs.readFileSync(path.join(testDataDir, mdFile), 'utf8');
expect(md).toContain('Hello Heading');
expect(md).toContain('Some paragraph text');
expect(md).not.toContain('should-be-stripped');