A free, self-hostable news aggregator…
Reduce memory consumption of export (#9350)
abp00248547b4a6c32
diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.phpindex 0daf66fc1..fc6d1e4db 100644--- a/app/Controllers/importExportController.php+++ b/app/Controllers/importExportController.php@@ -788,21 +788,21 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {$exported_files = [];if ($export_opml) {- [$filename, $content] = $export_service->generateOpml();- $exported_files[$filename] = $content;+ [$filename, $path] = $export_service->generateOpml();+ $exported_files[$filename] = $path;}// Starred and labelled entries are merged in the same `starred` file// to avoid duplication of content.if ($export_starred && $export_labelled) {- [$filename, $content] = $export_service->generateStarredEntries('ST');- $exported_files[$filename] = $content;+ [$filename, $path] = $export_service->generateStarredEntries('ST');+ $exported_files[$filename] = $path;} elseif ($export_starred) {- [$filename, $content] = $export_service->generateStarredEntries('S');- $exported_files[$filename] = $content;+ [$filename, $path] = $export_service->generateStarredEntries('S');+ $exported_files[$filename] = $path;} elseif ($export_labelled) {- [$filename, $content] = $export_service->generateStarredEntries('T');- $exported_files[$filename] = $content;+ [$filename, $path] = $export_service->generateStarredEntries('T');+ $exported_files[$filename] = $path;}foreach ($export_feeds as $feed_id) {@@ -812,8 +812,8 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {continue;}- [$filename, $content] = $result;- $exported_files[$filename] = $content;+ [$filename, $path] = $result;+ $exported_files[$filename] = $path;}$nb_files = count($exported_files);@@ -826,7 +826,7 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {if ($nb_files === 1) {// If we only have one file, we just export it as it is$filename = key($exported_files);- $content = $exported_files[$filename];+ $path = $exported_files[$filename];} else {// More files? Let’s compress them in a Zip archiveif (!extension_loaded('zip')) {@@ -838,10 +838,10 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {return;}- [$filename, $content] = $export_service->zip($exported_files);+ [$filename, $path] = $export_service->zip($exported_files);}- if (!is_string($content)) {+ if (!is_string($path)) {Minz_Request::bad(_t('feedback.import_export.zip_error'), ['c' => 'importExport', 'a' => 'index']);return;}@@ -851,7 +851,7 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {header('Content-disposition: attachment; filename="' . $filename . '"');$this->view->_layout(null);- $this->view->content = $content;+ $this->view->exportPath = $path;}/**diff --git a/app/Models/View.php b/app/Models/View.phpindex 43cf634d6..3df822230 100644--- a/app/Models/View.php+++ b/app/Models/View.php@@ -91,6 +91,7 @@ class FreshRSS_View extends Minz_View {// Export / Importpublic string $content;+ public string $exportPath;public int $feedCount;/** @var array<string,array<string>> */public array $entryIdsTagNames = [];diff --git a/app/Services/ExportService.php b/app/Services/ExportService.phpindex 325ff6322..c559b92d2 100644--- a/app/Services/ExportService.php+++ b/app/Services/ExportService.php@@ -40,8 +40,9 @@ class FreshRSS_Export_Service {}/**- * Generate OPML file content.- * @return array{0:string,1:string} First item is the filename, second item is the content+ * Generate OPML file.+ * @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content+ * @throws Minz_PermissionDeniedException*/public function generateOpml(): array {$view = new FreshRSS_View();@@ -52,12 +53,12 @@ class FreshRSS_Export_Service {return ["feeds_{$day}.opml.xml",- $view->helperToString('export/opml')+ self::helperToTempFile($view, 'export/opml'),];}/**- * Generate the starred and labelled entries file content.+ * Generate the starred and labelled entries file.** Both starred and labelled entries are put into a "starred" file, that’s* why there is only one method for both.@@ -67,7 +68,8 @@ class FreshRSS_Export_Service {* 'S' (starred/favourite),* 'T' (taggued/labelled),* 'ST' (starred or labelled)- * @return array{0:string,1:string} First item is the filename, second item is the content+ * @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content+ * @throws Minz_PermissionDeniedException*/public function generateStarredEntries(string $type): array {$view = new FreshRSS_View();@@ -85,14 +87,15 @@ class FreshRSS_Export_Service {return ["starred_{$day}.json",- $view->helperToString('export/articles')+ self::helperToTempFile($view, 'export/articles'),];}/**- * Generate the entries file content for the given feed.- * @return array{0:string,1:string}|null First item is the filename, second item is the content.+ * Generate the entries file for the given feed.+ * @return array{0:string,1:string}|null First item is the filename, second item is the path of a temporary file with the content.* It also can return null if the feed doesn’t exist.+ * @throws Minz_PermissionDeniedException*/public function generateFeedEntries(int $feed_id, int $max_number_entries): ?array {$view = new FreshRSS_View();@@ -120,13 +123,14 @@ class FreshRSS_Export_Service {return [$filename,- $view->helperToString('export/articles')+ self::helperToTempFile($view, 'export/articles'),];}/**- * Generate the entries file content for all the feeds.- * @return array<string,string> Keys are filenames and values are contents.+ * Generate the entries files for all the feeds.+ * @return array<string,string> Keys are filenames and values are paths of temporary files with the contents.+ * @throws Minz_PermissionDeniedException*/public function generateAllFeedEntries(int $max_number_entries): array {$feed_ids = $this->feed_dao->listFeedsIds();@@ -138,8 +142,8 @@ class FreshRSS_Export_Service {continue;}- [$filename, $content] = $result;- $exported_files[$filename] = $content;+ [$filename, $path] = $result;+ $exported_files[$filename] = $path;}return $exported_files;@@ -147,34 +151,49 @@ class FreshRSS_Export_Service {/*** Compress several files in a Zip file.- * @param array<string,string> $files where the key is the filename, the value is the content- * @return array{0:string,1:string|false} First item is the zip filename, second item is the zip content+ * @param array<string,string> $files where the key is the filename, the value is the path of the file+ * @return array{0:string,1:string|false} First item is the zip filename, second item is the path of a temporary zip file, or false in case of error+ * @throws Minz_PermissionDeniedException*/public function zip(array $files): array {$day = date('Y-m-d');$zip_filename = 'freshrss_' . $this->username . '_' . $day . '_export.zip';- // From https://stackoverflow.com/questions/1061710/php-zip-files-on-the-fly- $zip_file = tempnam(TMP_PATH, 'zip');- if ($zip_file === false) {- return [$zip_filename, false];- }+ $zip_file = self::tempFile();$zip_archive = new ZipArchive();$zip_archive->open($zip_file, ZipArchive::OVERWRITE);- foreach ($files as $filename => $content) {- $zip_archive->addFromString($filename, $content);+ foreach ($files as $filename => $path) {+ $zip_archive->addFile($path, $filename);}- $zip_archive->close();-- $content = file_get_contents($zip_file);-- unlink($zip_file);-return [$zip_filename,- $content,+ $zip_archive->close() ? $zip_file : false,];}++ /**+ * Create a temporary file, which is deleted at the end of the script.+ * @throws Minz_PermissionDeniedException+ */+ private static function tempFile(): string {+ $filename = tempnam(TMP_PATH, 'export');+ if ($filename === false) {+ throw new Minz_PermissionDeniedException(TMP_PATH);+ }+ register_shutdown_function(static fn() => @unlink($filename));+ return $filename;+ }++ /**+ * Render a view helper into a temporary file, instead of keeping the whole content in memory.+ * @return string The path of the temporary file+ * @throws Minz_PermissionDeniedException+ */+ private static function helperToTempFile(FreshRSS_View $view, string $helper): string {+ $filename = self::tempFile();+ $view->helperToFile($helper, $filename);+ return $filename;+ }}diff --git a/app/views/importExport/export.phtml b/app/views/importExport/export.phtmlindex af0b21ea1..3f78bbf37 100644--- a/app/views/importExport/export.phtml+++ b/app/views/importExport/export.phtml@@ -1,4 +1,5 @@<?phpdeclare(strict_types=1);/** @var FreshRSS_View $this */-echo $this->content;+@ob_end_clean(); // Ensure no buffer+readfile($this->exportPath);diff --git a/cli/export-opml-for-user.php b/cli/export-opml-for-user.phpindex 48925f1ff..78ef4f5af 100755--- a/cli/export-opml-for-user.php+++ b/cli/export-opml-for-user.php@@ -23,8 +23,8 @@ $username = cliInitUser($cliOptions->user);fwrite(STDERR, 'FreshRSS exporting OPML for user “' . $username . "”…\n");$export_service = new FreshRSS_Export_Service($username);-[$filename, $content] = $export_service->generateOpml();-echo $content;+[$filename, $path] = $export_service->generateOpml();+readfile($path);invalidateHttpCache($username);diff --git a/cli/export-zip-for-user.php b/cli/export-zip-for-user.phpindex ca7de28fe..1faeaf9a9 100755--- a/cli/export-zip-for-user.php+++ b/cli/export-zip-for-user.php@@ -33,12 +33,12 @@ $number_entries = $cliOptions->maxFeedEntries;$exported_files = [];// First, we generate the OPML file-[$filename, $content] = $export_service->generateOpml();-$exported_files[$filename] = $content;+[$filename, $path] = $export_service->generateOpml();+$exported_files[$filename] = $path;// Then, labelled and starred entries-[$filename, $content] = $export_service->generateStarredEntries('ST');-$exported_files[$filename] = $content;+[$filename, $path] = $export_service->generateStarredEntries('ST');+$exported_files[$filename] = $path;// And a list of entries based on the complete list of feeds$feeds_exported_files = $export_service->generateAllFeedEntries($number_entries);@@ -46,8 +46,11 @@ $exported_files = array_merge($exported_files, $feeds_exported_files);// Finally, we compress all these files into a single Zip archive and we output// the content-[$filename, $content] = $export_service->zip($exported_files);-echo $content;+[$filename, $path] = $export_service->zip($exported_files);+if ($path === false) {+ fail('FreshRSS error: cannot create the Zip archive!');+}+readfile($path);invalidateHttpCache($username);diff --git a/lib/Minz/View.php b/lib/Minz/View.phpindex 7dff550c1..74403fdd2 100644--- a/lib/Minz/View.php+++ b/lib/Minz/View.php@@ -157,6 +157,33 @@ class Minz_View {return ob_get_clean() ?: '';}+ /**+ * Writes renderHelper() into a file, without keeping the whole output in memory+ * @param string $helper the element to be treated+ * @param string $filename the path of the file to write+ * @throws Minz_PermissionDeniedException+ */+ public function helperToFile(string $helper, string $filename): void {+ $file = fopen($filename, 'wb');+ if ($file === false) {+ throw new Minz_PermissionDeniedException($filename);+ }+ $written = true;+ ob_start(static function (string $buffer) use ($file, &$written): string {+ $written = $written && fwrite($file, $buffer) !== false;+ return '';+ }, 65536); // Write to the file by chunks of 64 KiB+ try {+ $this->renderHelper($helper);+ } finally {+ ob_end_flush();+ $written = fclose($file) && $written;+ }+ if (!$written) {+ throw new Minz_PermissionDeniedException($filename);+ }+ }+/*** Choose the current view layout.* @param string|null $layout the layout name to use, null to use no layouts.diff --git a/p/api/greader.php b/p/api/greader.phpindex bd1eaf55b..bc627c15e 100644--- a/p/api/greader.php+++ b/p/api/greader.php@@ -315,10 +315,10 @@ final class GReaderAPI {private static function subscriptionExport(): never {$user = Minz_User::name() ?? Minz_User::INTERNAL_USER;$export_service = new FreshRSS_Export_Service($user);- [$filename, $content] = $export_service->generateOpml();+ [$filename, $path] = $export_service->generateOpml();header('Content-Type: application/xml; charset=UTF-8');header('Content-disposition: attachment; filename="' . $filename . '"');- echo $content;+ readfile($path);exit();}diff --git a/tests/app/Views/exportArticlesTest.php b/tests/app/Views/exportArticlesTest.phpnew file mode 100644index 000000000..6ba90fe6d--- /dev/null+++ b/tests/app/Views/exportArticlesTest.php@@ -0,0 +1,77 @@+<?php+declare(strict_types=1);++final class exportArticlesTest extends \PHPUnit\Framework\TestCase {+ private string $filename;++ #[\Override]+ public static function setUpBeforeClass(): void {+ // `FreshRSS_View` needs a system configuration; the shipped defaults are enough to render an export.+ Minz_Configuration::register('system', FRESHRSS_PATH . '/config.default.php', FRESHRSS_PATH . '/config.default.php');+ }++ #[\Override]+ protected function setUp(): void {+ $filename = tempnam(sys_get_temp_dir(), 'freshrss_test_');+ self::assertIsString($filename);+ $this->filename = $filename;+ }++ #[\Override]+ protected function tearDown(): void {+ @unlink($this->filename);+ }++ /** @return list<FreshRSS_Entry> */+ private static function entries(int $count): array {+ $entries = [];+ for ($i = 1; $i <= $count; $i++) {+ $entry = new FreshRSS_Entry(1, 'guid-' . $i, 'Title ' . $i, '', str_repeat('<p>Contenu exporté ' . $i . '</p>', 200),+ 'https://example.net/' . $i, 1700000000 + $i);+ $entry->_id(1700000000000000 + $i);+ $entries[] = $entry;+ }+ return $entries;+ }++ /** @param iterable<FreshRSS_Entry> $entries */+ private static function view(iterable $entries): FreshRSS_View {+ $view = new FreshRSS_View();+ $view->internal_rendering = true;+ $view->list_title = 'Starred';+ $view->type = 'starred';+ $view->feed = new FreshRSS_Feed('https://example.net/feed.xml', false);+ $view->entries = $entries;+ return $view;+ }++ public function testHelperToFileWritesTheSameContentAsHelperToString(): void {+ $expected = self::view(self::entries(50))->helperToString('export/articles');+ self::assertGreaterThan(65536, strlen($expected), 'The export must be larger than one chunk of the output buffer');++ file_put_contents($this->filename, 'Previous content, to be overwritten');+ $this->expectOutputString(''); // Nothing must leak to the output+ self::view(self::entries(50))->helperToFile('export/articles', $this->filename);++ self::assertSame($expected, file_get_contents($this->filename));+ $json = json_decode($expected, true);+ self::assertIsArray($json);+ self::assertIsArray($json['items'] ?? null);+ self::assertCount(50, $json['items']);+ }++ public function testHelperToFileClosesTheOutputBufferOnError(): void {+ $entries = (static function (): Generator {+ yield from self::entries(1);+ throw new RuntimeException('Database error');+ })();+ $level = ob_get_level();+ try {+ self::view($entries)->helperToFile('export/articles', $this->filename);+ self::fail('The exception must be propagated');+ } catch (RuntimeException $e) {+ self::assertSame('Database error', $e->getMessage());+ }+ self::assertSame($level, ob_get_level());+ }+}