TagSpaces is an offline, open source, document manager with tagging support
chore: added better cleaning of html documents
Ilian Sapundshievcfa762911a17
diff --git a/src/renderer/components/dialogs/DownloadUrlDialog.tsx b/src/renderer/components/dialogs/DownloadUrlDialog.tsxindex 28f733682..d924975d1 100644--- a/src/renderer/components/dialogs/DownloadUrlDialog.tsx+++ b/src/renderer/components/dialogs/DownloadUrlDialog.tsx@@ -130,6 +130,13 @@ function extForFormat(format: SaveFormat, originalExt: string): string {return originalExt || '.html';}+// Whether a (lower-cased) extension is an HTML document — these get sanitized+// (script/link stripped) even in "Original" mode; everything else (images, PDF,+// archives, …) is downloaded raw.+function isHtmlExtension(ext: string): boolean {+ return ['.html', '.htm', '.xhtml', ''].includes(ext.toLowerCase());+}+function DownloadUrlDialog(props: Props) {const { open, onClose } = props;const { t } = useTranslation();@@ -137,7 +144,8 @@ function DownloadUrlDialog(props: Props) {const smallScreen = useMediaQuery(theme.breakpoints.down('md'));const { setReflectActions } = useEditedEntryContext();const { currentLocation } = useCurrentLocationContext();- const { downloadUrl, downloadUrlAs } = useIOActionsContext();+ const { downloadUrl, downloadUrlAs, getUrlReaderable } =+ useIOActionsContext();const { showNotification } = useNotificationContext();const { openFileUploadDialog } = useFileUploadDialogContext();const dispatch: AppDispatch = useDispatch();@@ -189,6 +197,36 @@ function DownloadUrlDialog(props: Props) {// eslint-disable-next-line react-hooks/exhaustive-deps}, [open]);+ // When a conversion format is selected, default the "Extract main article"+ // switch to checked only if Readability considers the page a parseable+ // article. Fetches the page (debounced) and caches the verdict per URL.+ const readableCache = useRef<{ url: string; promise: Promise<boolean> }>();+ useEffect(() => {+ if (!open || saveFormat === 'original' || !/^https?:\/\//i.test(url)) {+ return undefined;+ }+ let cancelled = false;+ const timer = setTimeout(() => {+ if (!readableCache.current || readableCache.current.url !== url) {+ readableCache.current = { url, promise: getUrlReaderable(url) };+ }+ readableCache.current.promise+ .then((readerable) => {+ if (!cancelled) {+ setExtractArticle(readerable);+ }+ return readerable;+ })+ .catch(() => {});+ }, 500);+ return () => {+ cancelled = true;+ clearTimeout(timer);+ };+ // getUrlReaderable is a stable context fn.+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [url, saveFormat, open]);+const onUploadProgress = (progress, abort, fileName2) => {dispatch(AppActions.onUploadProgress(progress, abort, fileName2));};@@ -297,28 +335,50 @@ function DownloadUrlDialog(props: Props) {);};- if (canConvert && saveFormat !== 'original') {- // Cleaned-HTML / Markdown clip: fetch the page, strip scripts, inline- // images and optionally extract the article, then save into the location.- downloadUrlAs(url, targetPath, saveFormat, {+ const tagsCsv = tagTitles.join(', ');+ const isHtml = isHtmlExtension(splitNameExt(fileName).ext);++ if (!canConvert) {+ // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain+ // browser download (the menu entry is hidden on web anyway).+ saveAs(url, finalName);+ } else if (saveFormat === 'markdown') {+ // Convert the page to Markdown.+ downloadUrlAs(url, targetPath, 'markdown', {extractArticle,embedImages,- tags: tagTitles.join(', '),+ tags: tagsCsv,}).then(reflectOpen).catch(notifyError);- } else if (AppConfig.isElectron || AppConfig.isNativeMobile) {- // Raw download straight into the location (local + cloud + mobile),- // routed through the platform's CORS/CSP-free fetch.+ } else if (saveFormat === 'html') {+ // Cleaned HTML: strip scripts, links AND styles/CSS.+ downloadUrlAs(url, targetPath, 'html', {+ stripStyles: true,+ extractArticle,+ embedImages,+ tags: tagsCsv,+ })+ .then(reflectOpen)+ .catch(notifyError);+ } else if (isHtml) {+ // Original HTML: keep the page intact but strip <script> and <link>.+ downloadUrlAs(url, targetPath, 'html', {+ stripStyles: false,+ extractArticle: false,+ embedImages: false,+ tags: tagsCsv,+ })+ .then(reflectOpen)+ .catch(notifyError);+ } else {+ // Original non-HTML (image, PDF, archive, …): raw binary download via the+ // platform's CORS/CSP-free fetch, streamed straight into the location.dispatch(AppActions.resetProgress());openFileUploadDialog();downloadUrl(url, targetPath, onUploadProgress).then(reflectOpen).catch(notifyError);- } else {- // Web app: browser CSP/CORS cannot be bypassed — fall back to a plain- // browser download.- saveAs(url, finalName);}onClose();}diff --git a/src/renderer/hooks/EditedEntryContextProvider.tsx b/src/renderer/hooks/EditedEntryContextProvider.tsxindex 4bcbc8758..1516fe39d 100644--- a/src/renderer/hooks/EditedEntryContextProvider.tsx+++ b/src/renderer/hooks/EditedEntryContextProvider.tsx@@ -23,7 +23,10 @@ import React, {useReducer,useState,} from 'react';-import { extractTagsAsObjects } from '@tagspaces/tagspaces-common/paths';+import {+ extractTagsAsObjects,+ isMeta,+} from '@tagspaces/tagspaces-common/paths';import { TS } from '-/tagspaces.namespace';import AppConfig from '-/AppConfig';import { useSelector } from 'react-redux';@@ -77,7 +80,24 @@ export const EditedEntryContextProvider = ({setActions(undefined);return;}- const actionsArray: TS.EditAction[] = entries.map((entry) => ({+ // Internal .ts sidecar/thumbnail/revision files are never shown in any+ // view, so no consumer acts on their delete actions. Emitting them is+ // not only useless — it's harmful: deleteFile()/deleteEntries() reflect+ // the real file first and then reflect this meta cleanup. When the two+ // setActions() calls aren't separated by a macrotask (Capacitor resolves+ // a non-existent-file delete via a microtask, unlike Electron's IPC),+ // React 18 batches them and the meta-only update clobbers the real+ // file-delete — leaving the grid and the opened viewer stale until a+ // manual reload. Drop meta entries; if nothing user-visible remains,+ // don't touch `actions` at all (an empty/undefined update would clobber+ // the preceding real delete just the same).+ const visibleEntries = entries.filter(+ (entry) => entry && !isMeta(entry.path),+ );+ if (visibleEntries.length === 0) {+ return;+ }+ const actionsArray: TS.EditAction[] = visibleEntries.map((entry) => ({action: 'delete',entry,}));diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsxindex 0575678a9..4cf36b64e 100644--- a/src/renderer/hooks/IOActionsContextProvider.tsx+++ b/src/renderer/hooks/IOActionsContextProvider.tsx@@ -52,6 +52,7 @@ import {ClipOptions,htmlToCleanHtml,htmlToMarkdown,+ isHtmlReaderable,} from '-/services/web-content';import { TS } from '-/tagspaces.namespace';import { CommonLocation } from '-/utils/CommonLocation';@@ -139,8 +140,10 @@ type IOActionsContextData = {extractArticle?: boolean;embedImages?: boolean;tags?: string;+ stripStyles?: boolean;},) => Promise<TS.FileSystemEntry>;+ getUrlReaderable: (url: string) => Promise<boolean>;downloadFsEntry: (fsEntry: TS.FileSystemEntry) => void;uploadFilesAPI: (files: Array<any>,@@ -275,6 +278,7 @@ export const IOActionsContext = createContext<IOActionsContextData>({copyFiles: undefined,downloadUrl: undefined,downloadUrlAs: undefined,+ getUrlReaderable: undefined,downloadFsEntry: undefined,uploadFilesAPI: undefined,uploadMeta: undefined,@@ -1169,6 +1173,7 @@ export const IOActionsContextProvider = ({extractArticle?: boolean;embedImages?: boolean;tags?: string;+ stripStyles?: boolean;} = {},): Promise<TS.FileSystemEntry> {const opts: ClipOptions = {@@ -1177,6 +1182,7 @@ export const IOActionsContextProvider = ({extractArticle: options.extractArticle,embedImages: options.embedImages,tags: options.tags,+ stripStyles: options.stripStyles,scrappedOn: new Date().toISOString(),};return fetchPageHtml(url)@@ -1190,6 +1196,13 @@ export const IOActionsContextProvider = ({);}+ /** Fetch a page and report whether Readability treats it as a parseable article. */+ function getUrlReaderable(url: string): Promise<boolean> {+ return fetchPageHtml(url)+ .then((html) => isHtmlReaderable(html))+ .catch(() => false);+ }+function downloadFsEntry(fsEntry: TS.FileSystemEntry) {const loc = findLocation(fsEntry.locationID);if (!loc) {@@ -2860,6 +2873,7 @@ export const IOActionsContextProvider = ({copyFiles,downloadUrl,downloadUrlAs,+ getUrlReaderable,downloadFsEntry,uploadFilesAPI,uploadFiles,diff --git a/src/renderer/services/web-content.ts b/src/renderer/services/web-content.tsindex aecc1d52f..d052921a7 100644--- a/src/renderer/services/web-content.ts+++ b/src/renderer/services/web-content.ts@@ -43,6 +43,8 @@ export interface ClipOptions {tags?: string;/** ISO timestamp stamped into metadata (injected so the package stays pure). */scrappedOn?: string;+ /** Also remove `<style>` tags and inline `style=` attributes (full clean). */+ stripStyles?: boolean;}type ImageTask = () => Promise<void>;@@ -273,10 +275,22 @@ async function inlineImagesInMarkdown(// Core sanitize / extract pipeline// ---------------------------------------------------------------------------+/** True when Readability considers the page a parseable article. */+export function isHtmlReaderable(html: string): boolean {+ try {+ const doc = new DOMParser().parseFromString(html, 'text/html');+ return isProbablyReaderable(doc);+ } catch (e) {+ return false;+ }+}+/*** Parse raw HTML, optionally reduce it to the Readability article, and sanitize- * it (DOMPurify strips `<script>` and inline event handlers). Returns a fresh- * sanitized Document.+ * it. DOMPurify strips `<script>` and inline event handlers by default; we also+ * always drop `<link>` (external stylesheets/resources), and — when+ * `stripStyles` is set — `<style>` tags and inline `style=` attributes too.+ * Returns a fresh sanitized Document.*/function buildCleanDoc(html: string, opts: ClipOptions): Document {const parser = new DOMParser();@@ -302,7 +316,18 @@ function buildCleanDoc(html: string, opts: ClipOptions): Document {}}- const clean = DOMPurify.sanitize(workingHtml, { WHOLE_DOCUMENT: true });+ // DOMPurify drops <script> + event handlers by default; we always also drop+ // <link>, and for a full clean <style> tags + inline style attributes.+ const clean = opts.stripStyles+ ? DOMPurify.sanitize(workingHtml, {+ WHOLE_DOCUMENT: true,+ FORBID_TAGS: ['script', 'link', 'style'],+ FORBID_ATTR: ['style'],+ })+ : DOMPurify.sanitize(workingHtml, {+ WHOLE_DOCUMENT: true,+ FORBID_TAGS: ['script', 'link'],+ });return parser.parseFromString(clean, 'text/html');}diff --git a/tests/e2e/download-url.pw.e2e.js b/tests/e2e/download-url.pw.e2e.jsindex f38c00817..a89609978 100644--- a/tests/e2e/download-url.pw.e2e.js+++ b/tests/e2e/download-url.pw.e2e.js@@ -29,7 +29,11 @@ import { clearDataStorage, closeWelcomePlaywright } from './welcome.helpers';const SAMPLE_TXT = 'TagSpaces download URL test content';const PAGE_HTML = `<!DOCTYPE html><html>- <head><title>Clip Test Page</title></head>+ <head>+ <title>Clip Test Page</title>+ <link rel="stylesheet" href="/style.css" />+ <style>body { color: rebeccapurple; }</style>+ </head><body><h1>Hello Heading</h1><p>Some paragraph text for clipping.</p>@@ -49,6 +53,13 @@ async function expectEntryStartingWith(base, timeout = 20000) {});}+// Read the saved page[…].html back from disk (local locations only).+function readSavedPageHtml(dir) {+ const file = fs.readdirSync(dir).find((name) => /^page.*\.html$/.test(name));+ expect(file).toBeTruthy();+ return fs.readFileSync(path.join(dir, file), 'utf8');+}+async function openDownloadUrlDialog() {await clickOn('[data-tid=folderContainerOpenDirMenu]');await clickOnMenuOperation('newFromDownloadURLTID');@@ -158,4 +169,42 @@ test.describe('TST54 - Download from URL', () => {expect(md).not.toContain('<script');}});++ test('TST5422 - Original HTML strips script + link, keeps styles [electron]', async ({+ testDataDir,+ }) => {+ await openDownloadUrlDialog();+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');+ // Default "Original file" format → light sanitize for HTML pages.+ await clickOn('[data-tid=downloadFileUrlTID]');+ await expectEntryStartingWith('page');++ const html = readSavedPageHtml(testDataDir);+ expect(html).toContain('Hello Heading');+ // script + link removed…+ expect(html).not.toContain('<script');+ expect(html).not.toContain('should-be-stripped');+ expect(html).not.toContain('<link');+ // …but styles are kept in "Original" mode.+ expect(html).toContain('<style');+ expect(html).toContain('rebeccapurple');+ });++ test('TST5423 - Cleaned HTML also strips styles/CSS [electron]', async ({+ testDataDir,+ }) => {+ await openDownloadUrlDialog();+ await setInputValue('[data-tid=newUrlTID] input', baseUrl + '/page.html');+ await clickOn('[data-tid=downloadFormatHtmlTID]');+ await clickOn('[data-tid=downloadFileUrlTID]');+ await expectEntryStartingWith('page');++ const html = readSavedPageHtml(testDataDir);+ expect(html).toContain('Hello Heading');+ expect(html).not.toContain('<script');+ expect(html).not.toContain('<link');+ // cleaned strips <style> tags + inline CSS too.+ expect(html).not.toContain('<style');+ expect(html).not.toContain('rebeccapurple');+ });});