Skip to content

A free, self-hostable news aggregator…

Reduce memory consumption of export (#9350)

abp00248547b4a6c32
diff --git a/app/Controllers/importExportController.php b/app/Controllers/importExportController.php
index 0daf66fc1..fc6d1e4db 100644
--- a/app/Controllers/importExportController.php
+++ b/app/Controllers/importExportController.php
@@ -788,21 +788,21 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
$exported_files = [];
if ($export_opml) {
- [$filename, $content] = $export_service->generateOpml();
- $exported_files[$filename] = $content;
+ [$filename, $path] = $export_service->generateOpml();
+ $exported_files[$filename] = $path;
}
// Starred and labelled entries are merged in the same `starred` file
// to avoid duplication of content.
if ($export_starred && $export_labelled) {
- [$filename, $content] = $export_service->generateStarredEntries('ST');
- $exported_files[$filename] = $content;
+ [$filename, $path] = $export_service->generateStarredEntries('ST');
+ $exported_files[$filename] = $path;
} elseif ($export_starred) {
- [$filename, $content] = $export_service->generateStarredEntries('S');
- $exported_files[$filename] = $content;
+ [$filename, $path] = $export_service->generateStarredEntries('S');
+ $exported_files[$filename] = $path;
} elseif ($export_labelled) {
- [$filename, $content] = $export_service->generateStarredEntries('T');
- $exported_files[$filename] = $content;
+ [$filename, $path] = $export_service->generateStarredEntries('T');
+ $exported_files[$filename] = $path;
}
foreach ($export_feeds as $feed_id) {
@@ -812,8 +812,8 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
continue;
}
- [$filename, $content] = $result;
- $exported_files[$filename] = $content;
+ [$filename, $path] = $result;
+ $exported_files[$filename] = $path;
}
$nb_files = count($exported_files);
@@ -826,7 +826,7 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
if ($nb_files === 1) {
// If we only have one file, we just export it as it is
$filename = key($exported_files);
- $content = $exported_files[$filename];
+ $path = $exported_files[$filename];
} else {
// More files? Let’s compress them in a Zip archive
if (!extension_loaded('zip')) {
@@ -838,10 +838,10 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
return;
}
- [$filename, $content] = $export_service->zip($exported_files);
+ [$filename, $path] = $export_service->zip($exported_files);
}
- if (!is_string($content)) {
+ if (!is_string($path)) {
Minz_Request::bad(_t('feedback.import_export.zip_error'), ['c' => 'importExport', 'a' => 'index']);
return;
}
@@ -851,7 +851,7 @@ class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
header('Content-disposition: attachment; filename="' . $filename . '"');
$this->view->_layout(null);
- $this->view->content = $content;
+ $this->view->exportPath = $path;
}
/**
diff --git a/app/Models/View.php b/app/Models/View.php
index 43cf634d6..3df822230 100644
--- a/app/Models/View.php
+++ b/app/Models/View.php
@@ -91,6 +91,7 @@ class FreshRSS_View extends Minz_View {
// Export / Import
public string $content;
+ public string $exportPath;
public int $feedCount;
/** @var array<string,array<string>> */
public array $entryIdsTagNames = [];
diff --git a/app/Services/ExportService.php b/app/Services/ExportService.php
index 325ff6322..c559b92d2 100644
--- a/app/Services/ExportService.php
+++ b/app/Services/ExportService.php
@@ -40,8 +40,9 @@ class FreshRSS_Export_Service {
}
/**
- * Generate OPML file content.
- * @return array{0:string,1:string} First item is the filename, second item is the content
+ * Generate OPML file.
+ * @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content
+ * @throws Minz_PermissionDeniedException
*/
public function generateOpml(): array {
$view = new FreshRSS_View();
@@ -52,12 +53,12 @@ class FreshRSS_Export_Service {
return [
"feeds_{$day}.opml.xml",
- $view->helperToString('export/opml')
+ self::helperToTempFile($view, 'export/opml'),
];
}
/**
- * Generate the starred and labelled entries file content.
+ * Generate the starred and labelled entries file.
*
* Both starred and labelled entries are put into a "starred" file, that’s
* why there is only one method for both.
@@ -67,7 +68,8 @@ class FreshRSS_Export_Service {
* 'S' (starred/favourite),
* 'T' (taggued/labelled),
* 'ST' (starred or labelled)
- * @return array{0:string,1:string} First item is the filename, second item is the content
+ * @return array{0:string,1:string} First item is the filename, second item is the path of a temporary file with the content
+ * @throws Minz_PermissionDeniedException
*/
public function generateStarredEntries(string $type): array {
$view = new FreshRSS_View();
@@ -85,14 +87,15 @@ class FreshRSS_Export_Service {
return [
"starred_{$day}.json",
- $view->helperToString('export/articles')
+ self::helperToTempFile($view, 'export/articles'),
];
}
/**
- * Generate the entries file content for the given feed.
- * @return array{0:string,1:string}|null First item is the filename, second item is the content.
+ * Generate the entries file for the given feed.
+ * @return array{0:string,1:string}|null First item is the filename, second item is the path of a temporary file with the content.
* It also can return null if the feed doesn’t exist.
+ * @throws Minz_PermissionDeniedException
*/
public function generateFeedEntries(int $feed_id, int $max_number_entries): ?array {
$view = new FreshRSS_View();
@@ -120,13 +123,14 @@ class FreshRSS_Export_Service {
return [
$filename,
- $view->helperToString('export/articles')
+ self::helperToTempFile($view, 'export/articles'),
];
}
/**
- * Generate the entries file content for all the feeds.
- * @return array<string,string> Keys are filenames and values are contents.
+ * Generate the entries files for all the feeds.
+ * @return array<string,string> Keys are filenames and values are paths of temporary files with the contents.
+ * @throws Minz_PermissionDeniedException
*/
public function generateAllFeedEntries(int $max_number_entries): array {
$feed_ids = $this->feed_dao->listFeedsIds();
@@ -138,8 +142,8 @@ class FreshRSS_Export_Service {
continue;
}
- [$filename, $content] = $result;
- $exported_files[$filename] = $content;
+ [$filename, $path] = $result;
+ $exported_files[$filename] = $path;
}
return $exported_files;
@@ -147,34 +151,49 @@ class FreshRSS_Export_Service {
/**
* Compress several files in a Zip file.
- * @param array<string,string> $files where the key is the filename, the value is the content
- * @return array{0:string,1:string|false} First item is the zip filename, second item is the zip content
+ * @param array<string,string> $files where the key is the filename, the value is the path of the file
+ * @return array{0:string,1:string|false} First item is the zip filename, second item is the path of a temporary zip file, or false in case of error
+ * @throws Minz_PermissionDeniedException
*/
public function zip(array $files): array {
$day = date('Y-m-d');
$zip_filename = 'freshrss_' . $this->username . '_' . $day . '_export.zip';
- // From https://stackoverflow.com/questions/1061710/php-zip-files-on-the-fly
- $zip_file = tempnam(TMP_PATH, 'zip');
- if ($zip_file === false) {
- return [$zip_filename, false];
- }
+ $zip_file = self::tempFile();
$zip_archive = new ZipArchive();
$zip_archive->open($zip_file, ZipArchive::OVERWRITE);
- foreach ($files as $filename => $content) {
- $zip_archive->addFromString($filename, $content);
+ foreach ($files as $filename => $path) {
+ $zip_archive->addFile($path, $filename);
}
- $zip_archive->close();
-
- $content = file_get_contents($zip_file);
-
- unlink($zip_file);
-
return [
$zip_filename,
- $content,
+ $zip_archive->close() ? $zip_file : false,
];
}
+
+ /**
+ * Create a temporary file, which is deleted at the end of the script.
+ * @throws Minz_PermissionDeniedException
+ */
+ private static function tempFile(): string {
+ $filename = tempnam(TMP_PATH, 'export');
+ if ($filename === false) {
+ throw new Minz_PermissionDeniedException(TMP_PATH);
+ }
+ register_shutdown_function(static fn() => @unlink($filename));
+ return $filename;
+ }
+
+ /**
+ * Render a view helper into a temporary file, instead of keeping the whole content in memory.
+ * @return string The path of the temporary file
+ * @throws Minz_PermissionDeniedException
+ */
+ private static function helperToTempFile(FreshRSS_View $view, string $helper): string {
+ $filename = self::tempFile();
+ $view->helperToFile($helper, $filename);
+ return $filename;
+ }
}
diff --git a/app/views/importExport/export.phtml b/app/views/importExport/export.phtml
index af0b21ea1..3f78bbf37 100644
--- a/app/views/importExport/export.phtml
+++ b/app/views/importExport/export.phtml
@@ -1,4 +1,5 @@
<?php
declare(strict_types=1);
/** @var FreshRSS_View $this */
-echo $this->content;
+@ob_end_clean(); // Ensure no buffer
+readfile($this->exportPath);
diff --git a/cli/export-opml-for-user.php b/cli/export-opml-for-user.php
index 48925f1ff..78ef4f5af 100755
--- a/cli/export-opml-for-user.php
+++ b/cli/export-opml-for-user.php
@@ -23,8 +23,8 @@ $username = cliInitUser($cliOptions->user);
fwrite(STDERR, 'FreshRSS exporting OPML for user “' . $username . "”…\n");
$export_service = new FreshRSS_Export_Service($username);
-[$filename, $content] = $export_service->generateOpml();
-echo $content;
+[$filename, $path] = $export_service->generateOpml();
+readfile($path);
invalidateHttpCache($username);
diff --git a/cli/export-zip-for-user.php b/cli/export-zip-for-user.php
index ca7de28fe..1faeaf9a9 100755
--- a/cli/export-zip-for-user.php
+++ b/cli/export-zip-for-user.php
@@ -33,12 +33,12 @@ $number_entries = $cliOptions->maxFeedEntries;
$exported_files = [];
// First, we generate the OPML file
-[$filename, $content] = $export_service->generateOpml();
-$exported_files[$filename] = $content;
+[$filename, $path] = $export_service->generateOpml();
+$exported_files[$filename] = $path;
// Then, labelled and starred entries
-[$filename, $content] = $export_service->generateStarredEntries('ST');
-$exported_files[$filename] = $content;
+[$filename, $path] = $export_service->generateStarredEntries('ST');
+$exported_files[$filename] = $path;
// And a list of entries based on the complete list of feeds
$feeds_exported_files = $export_service->generateAllFeedEntries($number_entries);
@@ -46,8 +46,11 @@ $exported_files = array_merge($exported_files, $feeds_exported_files);
// Finally, we compress all these files into a single Zip archive and we output
// the content
-[$filename, $content] = $export_service->zip($exported_files);
-echo $content;
+[$filename, $path] = $export_service->zip($exported_files);
+if ($path === false) {
+ fail('FreshRSS error: cannot create the Zip archive!');
+}
+readfile($path);
invalidateHttpCache($username);
diff --git a/lib/Minz/View.php b/lib/Minz/View.php
index 7dff550c1..74403fdd2 100644
--- a/lib/Minz/View.php
+++ b/lib/Minz/View.php
@@ -157,6 +157,33 @@ class Minz_View {
return ob_get_clean() ?: '';
}
+ /**
+ * Writes renderHelper() into a file, without keeping the whole output in memory
+ * @param string $helper the element to be treated
+ * @param string $filename the path of the file to write
+ * @throws Minz_PermissionDeniedException
+ */
+ public function helperToFile(string $helper, string $filename): void {
+ $file = fopen($filename, 'wb');
+ if ($file === false) {
+ throw new Minz_PermissionDeniedException($filename);
+ }
+ $written = true;
+ ob_start(static function (string $buffer) use ($file, &$written): string {
+ $written = $written && fwrite($file, $buffer) !== false;
+ return '';
+ }, 65536); // Write to the file by chunks of 64 KiB
+ try {
+ $this->renderHelper($helper);
+ } finally {
+ ob_end_flush();
+ $written = fclose($file) && $written;
+ }
+ if (!$written) {
+ throw new Minz_PermissionDeniedException($filename);
+ }
+ }
+
/**
* Choose the current view layout.
* @param string|null $layout the layout name to use, null to use no layouts.
diff --git a/p/api/greader.php b/p/api/greader.php
index bd1eaf55b..bc627c15e 100644
--- a/p/api/greader.php
+++ b/p/api/greader.php
@@ -315,10 +315,10 @@ final class GReaderAPI {
private static function subscriptionExport(): never {
$user = Minz_User::name() ?? Minz_User::INTERNAL_USER;
$export_service = new FreshRSS_Export_Service($user);
- [$filename, $content] = $export_service->generateOpml();
+ [$filename, $path] = $export_service->generateOpml();
header('Content-Type: application/xml; charset=UTF-8');
header('Content-disposition: attachment; filename="' . $filename . '"');
- echo $content;
+ readfile($path);
exit();
}
diff --git a/tests/app/Views/exportArticlesTest.php b/tests/app/Views/exportArticlesTest.php
new file mode 100644
index 000000000..6ba90fe6d
--- /dev/null
+++ b/tests/app/Views/exportArticlesTest.php
@@ -0,0 +1,77 @@
+<?php
+declare(strict_types=1);
+
+final class exportArticlesTest extends \PHPUnit\Framework\TestCase {
+ private string $filename;
+
+ #[\Override]
+ public static function setUpBeforeClass(): void {
+ // `FreshRSS_View` needs a system configuration; the shipped defaults are enough to render an export.
+ Minz_Configuration::register('system', FRESHRSS_PATH . '/config.default.php', FRESHRSS_PATH . '/config.default.php');
+ }
+
+ #[\Override]
+ protected function setUp(): void {
+ $filename = tempnam(sys_get_temp_dir(), 'freshrss_test_');
+ self::assertIsString($filename);
+ $this->filename = $filename;
+ }
+
+ #[\Override]
+ protected function tearDown(): void {
+ @unlink($this->filename);
+ }
+
+ /** @return list<FreshRSS_Entry> */
+ private static function entries(int $count): array {
+ $entries = [];
+ for ($i = 1; $i <= $count; $i++) {
+ $entry = new FreshRSS_Entry(1, 'guid-' . $i, 'Title ' . $i, '', str_repeat('<p>Contenu exporté ' . $i . '</p>', 200),
+ 'https://example.net/' . $i, 1700000000 + $i);
+ $entry->_id(1700000000000000 + $i);
+ $entries[] = $entry;
+ }
+ return $entries;
+ }
+
+ /** @param iterable<FreshRSS_Entry> $entries */
+ private static function view(iterable $entries): FreshRSS_View {
+ $view = new FreshRSS_View();
+ $view->internal_rendering = true;
+ $view->list_title = 'Starred';
+ $view->type = 'starred';
+ $view->feed = new FreshRSS_Feed('https://example.net/feed.xml', false);
+ $view->entries = $entries;
+ return $view;
+ }
+
+ public function testHelperToFileWritesTheSameContentAsHelperToString(): void {
+ $expected = self::view(self::entries(50))->helperToString('export/articles');
+ self::assertGreaterThan(65536, strlen($expected), 'The export must be larger than one chunk of the output buffer');
+
+ file_put_contents($this->filename, 'Previous content, to be overwritten');
+ $this->expectOutputString(''); // Nothing must leak to the output
+ self::view(self::entries(50))->helperToFile('export/articles', $this->filename);
+
+ self::assertSame($expected, file_get_contents($this->filename));
+ $json = json_decode($expected, true);
+ self::assertIsArray($json);
+ self::assertIsArray($json['items'] ?? null);
+ self::assertCount(50, $json['items']);
+ }
+
+ public function testHelperToFileClosesTheOutputBufferOnError(): void {
+ $entries = (static function (): Generator {
+ yield from self::entries(1);
+ throw new RuntimeException('Database error');
+ })();
+ $level = ob_get_level();
+ try {
+ self::view($entries)->helperToFile('export/articles', $this->filename);
+ self::fail('The exception must be propagated');
+ } catch (RuntimeException $e) {
+ self::assertSame('Database error', $e->getMessage());
+ }
+ self::assertSame($level, ob_get_level());
+ }
+}