Skip to content

TagSpaces is an offline, open source, document manager with tagging support

chore: added better cleaning of html documents

Ilian Sapundshievcfa762911a17
diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
index 28f733682..d924975d1 100644
--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx
+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx
@@ -130,6 +130,13 @@ function extForFormat(format: SaveFormat, originalExt: string): string {
return originalExt || '.html';
}
+// Whether a (lower-cased) extension is an HTML document — these get sanitized
+// (script/link stripped) even in "Original" mode; everything else (images, PDF,
+// archives, …) is downloaded raw.
+function isHtmlExtension(ext: string): boolean {
+ return ['.html', '.htm', '.xhtml', ''].includes(ext.toLowerCase());
+}
+
function DownloadUrlDialog(props: Props) {
const { open, onClose } = props;
const { t } = useTranslation();
@@ -137,7 +144,8 @@ function DownloadUrlDialog(props: Props) {
const smallScreen = useMediaQuery(theme.breakpoints.down('md'));
const { setReflectActions } = useEditedEntryContext();
const { currentLocation } = useCurrentLocationContext();
- const { downloadUrl, downloadUrlAs } = useIOActionsContext();
+ const { downloadUrl, downloadUrlAs, getUrlReaderable } =
+ useIOActionsContext();
const { showNotification } = useNotificationContext();
const { openFileUploadDialog } = useFileUploadDialogContext();
const dispatch: AppDispatch = useDispatch();
@@ -189,6 +197,36 @@ function DownloadUrlDialog(props: Props) {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [open]);
+ // When a conversion format is selected, default the "Extract main article"
+ // switch to checked only if Readability considers the page a parseable
+ // article. Fetches the page (debounced) and caches the verdict per URL.
+ const readableCache = useRef<{ url: string; promise: Promise<boolean> }>();
+ useEffect(() => {
+ if (!open || saveFormat === 'original' || !/^https?:\/\//i.test(url)) {
+ return undefined;
+ }
+ let cancelled = false;
+ const timer = setTimeout(() => {
+ if (!readableCache.current || readableCache.current.url !== url) {
+ readableCache.current = { url, promise: getUrlReaderable(url) };
+ }
+ readableCache.current.promise
+ .then((readerable) => {
+ if (!cancelled) {
+ setExtractArticle(readerable);
+ }
+ return readerable;
+ })
+ .catch(() => {});
+ }, 500);
+ return () => {
+ cancelled = true;
+ clearTimeout(timer);
+ };
+ // getUrlReaderable is a stable context fn.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [url, saveFormat, open]);
+
const onUploadProgress = (progress, abort, fileName2) => {
dispatch(AppActions.onUploadProgress(progress, abort, fileName2));
};
@@ -297,28 +335,50 @@ function DownloadUrlDialog(props: Props) {
);
};
- if (canConvert && saveFormat !== 'original') {
- // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline
- // images and optionally extract the article, then save into the location.
- downloadUrlAs(url, targetPath, saveFormat, {
+ const tagsCsv = tagTitles.join(', ');
+ const isHtml = isHtmlExtension(splitNameExt(fileName).ext);
+
+ if (!canConvert) {
+ // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain
+ // browser download (the menu entry is hidden on web anyway).
+ saveAs(url, finalName);
+ } else if (saveFormat === 'markdown') {
+ // Convert the page to Markdown.
+ downloadUrlAs(url, targetPath, 'markdown', {
extractArticle,
embedImages,
- tags: tagTitles.join(', '),
+ tags: tagsCsv,
})
.then(reflectOpen)
.catch(notifyError);
- } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {
- // Raw download straight into the location (local + cloud + mobile),
- // routed through the platform's CORS/CSP-free fetch.
+ } else if (saveFormat === 'html') {
+ // Cleaned HTML: strip scripts, links AND styles/CSS.
+ downloadUrlAs(url, targetPath, 'html', {
+ stripStyles: true,
+ extractArticle,
+ embedImages,
+ tags: tagsCsv,
+ })
+ .then(reflectOpen)
+ .catch(notifyError);
+ } else if (isHtml) {
+ // Original HTML: keep the page intact but strip <script> and <link>.
+ downloadUrlAs(url, targetPath, 'html', {
+ stripStyles: false,
+ extractArticle: false,
+ embedImages: false,
+ tags: tagsCsv,
+ })
+ .then(reflectOpen)
+ .catch(notifyError);
+ } else {
+ // Original non-HTML (image, PDF, archive, …): raw binary download via the
+ // platform's CORS/CSP-free fetch, streamed straight into the location.
dispatch(AppActions.resetProgress());
openFileUploadDialog();
downloadUrl(url, targetPath, onUploadProgress)
.then(reflectOpen)
.catch(notifyError);
- } else {
- // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain
- // browser download.
- saveAs(url, finalName);
}
onClose();
}
diff --git a/src/renderer/hooks/EditedEntryContextProvider.tsx b/src/renderer/hooks/EditedEntryContextProvider.tsx
index 4bcbc8758..1516fe39d 100644
--- a/src/renderer/hooks/EditedEntryContextProvider.tsx
+++ b/src/renderer/hooks/EditedEntryContextProvider.tsx
@@ -23,7 +23,10 @@ import React, {
useReducer,
useState,
} from 'react';
-import { extractTagsAsObjects } from '@tagspaces/tagspaces-common/paths';
+import {
+ extractTagsAsObjects,
+ isMeta,
+} from '@tagspaces/tagspaces-common/paths';
import { TS } from '-/tagspaces.namespace';
import AppConfig from '-/AppConfig';
import { useSelector } from 'react-redux';
@@ -77,7 +80,24 @@ export const EditedEntryContextProvider = ({
setActions(undefined);
return;
}
- const actionsArray: TS.EditAction[] = entries.map((entry) => ({
+ // Internal .ts sidecar/thumbnail/revision files are never shown in any
+ // view, so no consumer acts on their delete actions. Emitting them is
+ // not only useless — it's harmful: deleteFile()/deleteEntries() reflect
+ // the real file first and then reflect this meta cleanup. When the two
+ // setActions() calls aren't separated by a macrotask (Capacitor resolves
+ // a non-existent-file delete via a microtask, unlike Electron's IPC),
+ // React 18 batches them and the meta-only update clobbers the real
+ // file-delete — leaving the grid and the opened viewer stale until a
+ // manual reload. Drop meta entries; if nothing user-visible remains,
+ // don't touch `actions` at all (an empty/undefined update would clobber
+ // the preceding real delete just the same).
+ const visibleEntries = entries.filter(
+ (entry) => entry && !isMeta(entry.path),
+ );
+ if (visibleEntries.length === 0) {
+ return;
+ }
+ const actionsArray: TS.EditAction[] = visibleEntries.map((entry) => ({
action: 'delete',
entry,
}));
diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsx
index 0575678a9..4cf36b64e 100644
--- a/src/renderer/hooks/IOActionsContextProvider.tsx
+++ b/src/renderer/hooks/IOActionsContextProvider.tsx
@@ -52,6 +52,7 @@ import {
ClipOptions,
htmlToCleanHtml,
htmlToMarkdown,
+ isHtmlReaderable,
} from '-/services/web-content';
import { TS } from '-/tagspaces.namespace';
import { CommonLocation } from '-/utils/CommonLocation';
@@ -139,8 +140,10 @@ type IOActionsContextData = {
extractArticle?: boolean;
embedImages?: boolean;
tags?: string;
+ stripStyles?: boolean;
},
) => Promise<TS.FileSystemEntry>;
+ getUrlReaderable: (url: string) => Promise<boolean>;
downloadFsEntry: (fsEntry: TS.FileSystemEntry) => void;
uploadFilesAPI: (
files: Array<any>,
@@ -275,6 +278,7 @@ export const IOActionsContext = createContext<IOActionsContextData>({
copyFiles: undefined,
downloadUrl: undefined,
downloadUrlAs: undefined,
+ getUrlReaderable: undefined,
downloadFsEntry: undefined,
uploadFilesAPI: undefined,
uploadMeta: undefined,
@@ -1169,6 +1173,7 @@ export const IOActionsContextProvider = ({
extractArticle?: boolean;
embedImages?: boolean;
tags?: string;
+ stripStyles?: boolean;
} = {},
): Promise<TS.FileSystemEntry> {
const opts: ClipOptions = {
@@ -1177,6 +1182,7 @@ export const IOActionsContextProvider = ({
extractArticle: options.extractArticle,
embedImages: options.embedImages,
tags: options.tags,
+ stripStyles: options.stripStyles,
scrappedOn: new Date().toISOString(),
};
return fetchPageHtml(url)
@@ -1190,6 +1196,13 @@ export const IOActionsContextProvider = ({
);
}
+ /** Fetch a page and report whether Readability treats it as a parseable article. */
+ function getUrlReaderable(url: string): Promise<boolean> {
+ return fetchPageHtml(url)
+ .then((html) => isHtmlReaderable(html))
+ .catch(() => false);
+ }
+
function downloadFsEntry(fsEntry: TS.FileSystemEntry) {
const loc = findLocation(fsEntry.locationID);
if (!loc) {
@@ -2860,6 +2873,7 @@ export const IOActionsContextProvider = ({
copyFiles,
downloadUrl,
downloadUrlAs,
+ getUrlReaderable,
downloadFsEntry,
uploadFilesAPI,
uploadFiles,
diff --git a/src/renderer/services/web-content.ts b/src/renderer/services/web-content.ts
index aecc1d52f..d052921a7 100644
--- a/src/renderer/services/web-content.ts
+++ b/src/renderer/services/web-content.ts
@@ -43,6 +43,8 @@ export interface ClipOptions {
tags?: string;
/** ISO timestamp stamped into metadata (injected so the package stays pure). */
scrappedOn?: string;
+ /** Also remove `<style>` tags and inline `style=` attributes (full clean). */
+ stripStyles?: boolean;
}
type ImageTask = () => Promise<void>;
@@ -273,10 +275,22 @@ async function inlineImagesInMarkdown(
// Core sanitize / extract pipeline
// ---------------------------------------------------------------------------
+/** True when Readability considers the page a parseable article. */
+export function isHtmlReaderable(html: string): boolean {
+ try {
+ const doc = new DOMParser().parseFromString(html, 'text/html');
+ return isProbablyReaderable(doc);
+ } catch (e) {
+ return false;
+ }
+}
+
/**
* Parse raw HTML, optionally reduce it to the Readability article, and sanitize
- * it (DOMPurify strips `<script>` and inline event handlers). Returns a fresh
- * sanitized Document.
+ * it. DOMPurify strips `<script>` and inline event handlers by default; we also
+ * always drop `<link>` (external stylesheets/resources), and — when
+ * `stripStyles` is set — `<style>` tags and inline `style=` attributes too.
+ * Returns a fresh sanitized Document.
*/
function buildCleanDoc(html: string, opts: ClipOptions): Document {
const parser = new DOMParser();
@@ -302,7 +316,18 @@ function buildCleanDoc(html: string, opts: ClipOptions): Document {
}
}
- const clean = DOMPurify.sanitize(workingHtml, { WHOLE_DOCUMENT: true });
+ // DOMPurify drops <script> + event handlers by default; we always also drop
+ // <link>, and for a full clean <style> tags + inline style attributes.
+ const clean = opts.stripStyles
+ ? DOMPurify.sanitize(workingHtml, {
+ WHOLE_DOCUMENT: true,
+ FORBID_TAGS: ['script', 'link', 'style'],
+ FORBID_ATTR: ['style'],
+ })
+ : DOMPurify.sanitize(workingHtml, {
+ WHOLE_DOCUMENT: true,
+ FORBID_TAGS: ['script', 'link'],
+ });
return parser.parseFromString(clean, 'text/html');
}
diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.js
index f38c00817..a89609978 100644
--- a/tests/e2e/download-url.pw.e2e.js
+++ b/tests/e2e/download-url.pw.e2e.js
@@ -29,7 +29,11 @@ import { clearDataStorage, closeWelcomePlaywright } from './welcome.helpers';
const SAMPLE_TXT = 'TagSpaces download URL test content';
const PAGE_HTML = `<!DOCTYPE html>
<html>
- <head><title>Clip Test Page</title></head>
+ <head>
+ <title>Clip Test Page</title>
+ <link rel="stylesheet" href="/style.css" />
+ <style>body { color: rebeccapurple; }</style>
+ </head>
<body>
<h1>Hello Heading</h1>
<p>Some paragraph text for clipping.</p>
@@ -49,6 +53,13 @@ async function expectEntryStartingWith(base, timeout = 20000) {
});
}
+// Read the saved page[…].html back from disk (local locations only).
+function readSavedPageHtml(dir) {
+ const file = fs.readdirSync(dir).find((name) => /^page.*\.html$/.test(name));
+ expect(file).toBeTruthy();
+ return fs.readFileSync(path.join(dir, file), 'utf8');
+}
+
async function openDownloadUrlDialog() {
await clickOn('[data-tid=folderContainerOpenDirMenu]');
await clickOnMenuOperation('newFromDownloadURLTID');
@@ -158,4 +169,42 @@ test.describe('TST54 - Download from URL', () => {
expect(md).not.toContain('<script');
}
});
+
+ test('TST5422 - Original HTML strips script + link, keeps styles [electron]', async ({
+ testDataDir,
+ }) => {
+ await openDownloadUrlDialog();
+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');
+ // Default "Original file" format → light sanitize for HTML pages.
+ await clickOn('[data-tid=downloadFileUrlTID]');
+ await expectEntryStartingWith('page');
+
+ const html = readSavedPageHtml(testDataDir);
+ expect(html).toContain('Hello Heading');
+ // script + link removed…
+ expect(html).not.toContain('<script');
+ expect(html).not.toContain('should-be-stripped');
+ expect(html).not.toContain('<link');
+ // …but styles are kept in "Original" mode.
+ expect(html).toContain('<style');
+ expect(html).toContain('rebeccapurple');
+ });
+
+ test('TST5423 - Cleaned HTML also strips styles/CSS [electron]', async ({
+ testDataDir,
+ }) => {
+ await openDownloadUrlDialog();
+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');
+ await clickOn('[data-tid=downloadFormatHtmlTID]');
+ await clickOn('[data-tid=downloadFileUrlTID]');
+ await expectEntryStartingWith('page');
+
+ const html = readSavedPageHtml(testDataDir);
+ expect(html).toContain('Hello Heading');
+ expect(html).not.toContain('<script');
+ expect(html).not.toContain('<link');
+ // cleaned strips <style> tags + inline CSS too.
+ expect(html).not.toContain('<style');
+ expect(html).not.toContain('rebeccapurple');
+ });
});