A free open source IT asset/license management system
Use chunk by ID in purge for larger data sets
snipeedc57459cb15
diff --git a/app/Console/Commands/Purge.php b/app/Console/Commands/Purge.phpindex dd36e13282..bc265a82f9 100644--- a/app/Console/Commands/Purge.php+++ b/app/Console/Commands/Purge.php@@ -187,16 +187,19 @@ class Purge extends Command// Polymorphic action_log cleanup. Every model referenced by// action_logs uses one of two column pairs. Users use target_*,- // everything else uses item_*.+ // everything else uses item_*. Chunked whereIn turns what used+ // to be N*2 individual DELETEs into (N/1000)*2 bulk ones. Fixes+ // the big-purge timeout + memory bomb on sites with hundreds+ // of thousands of trashed rows.$itemLogs = 0;$targetLogs = 0;- foreach ($ids as $id) {+ foreach ($ids->chunk(1000) as $chunk) {$itemQuery = DB::table('action_logs')->where('item_type', $modelClass)- ->where('item_id', $id);+ ->whereIn('item_id', $chunk);$targetQuery = DB::table('action_logs')->where('target_type', $modelClass)- ->where('target_id', $id);+ ->whereIn('target_id', $chunk);if ($dryRun) {$itemLogs += $itemQuery->count();$targetLogs += $targetQuery->count();@@ -258,17 +261,20 @@ class Purge extends CommandComponent::class => ['checkout_requests' => ['requestable_id', 'requestable_type', Component::class]],AssetModel::class => ['checkout_requests' => ['requestable_id', 'requestable_type', AssetModel::class]],];+ // Chunked whereIn, same pattern as the action_log cleanup+ // above. Replaces N*k individual DELETEs (k = number of+ // child-table entries for this model) with (N/1000)*k bulk ones.$childCounts = [];if (array_key_exists($modelClass, $childTables)) {foreach ($childTables[$modelClass] as $childTable => $foreignKey) {$count = 0;- foreach ($ids as $id) {+ foreach ($ids->chunk(1000) as $chunk) {$q = DB::table($childTable);if (is_array($foreignKey)) {[$idColumn, $typeColumn, $expectedType] = $foreignKey;- $q->where($idColumn, $id)->where($typeColumn, $expectedType);+ $q->whereIn($idColumn, $chunk)->where($typeColumn, $expectedType);} else {- $q->where($foreignKey, $id);+ $q->whereIn($foreignKey, $chunk);}$count += $dryRun ? $q->count() : $q->delete();}@@ -394,43 +400,56 @@ class Purge extends Command}}- // Chunk the IN list so a user with tens of thousands of- // trashed rows doesn't blow MySQL's placeholder ceiling- // on a single prepared statement. Collected into one $logs- // collection so the dedupe + unlink step below stays reasonable.- $logs = $ids->chunk(1000)->flatMap(fn ($chunk) => DB::table('action_logs')- ->select('action_type', 'item_type', 'filename', 'accept_signature')- ->where(function ($outer) use ($modelClass, $chunk) {- $outer->where(function ($s) use ($modelClass, $chunk) {- $s->where('item_type', $modelClass)->whereIn('item_id', $chunk);- })->orWhere(function ($s) use ($modelClass, $chunk) {- $s->where('target_type', $modelClass)->whereIn('target_id', $chunk);+ // Chunk the parent ids AND stream the matching action_logs via+ // chunkById so only one page of logs lives in memory at a time.+ // The old shape collected every log row into a single Collection+ // (via flatMap(...)->get()) and OOM'd a web request on sites+ // with hundreds of thousands of rows per model. Each chunk's+ // paths unlink immediately before the next chunk loads.+ //+ // $seenPaths dedupes across chunks so the same signature or+ // EULA file (referenced by multiple logs) isn't unlinked twice.+ $seenPaths = [];+ foreach ($ids->chunk(1000) as $idChunk) {+ DB::table('action_logs')+ ->select('id', 'action_type', 'item_type', 'filename', 'accept_signature')+ ->where(function ($outer) use ($modelClass, $idChunk) {+ $outer->where(function ($s) use ($modelClass, $idChunk) {+ $s->where('item_type', $modelClass)->whereIn('item_id', $idChunk);+ })->orWhere(function ($s) use ($modelClass, $idChunk) {+ $s->where('target_type', $modelClass)->whereIn('target_id', $idChunk);+ });+ })+ ->orderBy('id')+ ->chunkById(1000, function ($logs) use (&$seenPaths, $uploadRoots) {+ foreach ($logs as $log) {+ // Map action_type to disk path for the attached file.+ // Mirrors Actionlog::uploads_file_path() but works+ // off raw query-builder rows (no Eloquent).+ if (! empty($log->filename)) {+ if ($log->action_type === ActionType::Accepted->value || $log->action_type === ActionType::Declined->value) {+ $path = FileStorage::EulaPdfs->privateStorageKey().$log->filename;+ } elseif ($log->action_type === ActionType::Audit->value) {+ $path = FileStorage::Audits->privateStorageKey().$log->filename;+ } elseif ($log->item_type && isset($uploadRoots[$log->item_type])) {+ $path = $uploadRoots[$log->item_type].$log->filename;+ } else {+ $path = null;+ }+ if ($path !== null && ! isset($seenPaths[$path])) {+ $seenPaths[$path] = true;+ $this->tryUnlink($path);+ }+ }+ if (! empty($log->accept_signature)) {+ $sigPath = FileStorage::Signatures->privateStorageKey().$log->accept_signature;+ if (! isset($seenPaths[$sigPath])) {+ $seenPaths[$sigPath] = true;+ $this->tryUnlink($sigPath);+ }+ }+ }});- })- ->get());-- $paths = [];- foreach ($logs as $log) {- // Map action_type to disk path for the attached file, or- // skip if the log carries no attachment we know how to route.- // Mirrors Actionlog::uploads_file_path() but works off raw- // query-builder rows (no Eloquent).- if (! empty($log->filename)) {- if ($log->action_type === ActionType::Accepted->value || $log->action_type === ActionType::Declined->value) {- $paths[] = FileStorage::EulaPdfs->privateStorageKey().$log->filename;- } elseif ($log->action_type === ActionType::Audit->value) {- $paths[] = FileStorage::Audits->privateStorageKey().$log->filename;- } elseif ($log->item_type && isset($uploadRoots[$log->item_type])) {- $paths[] = $uploadRoots[$log->item_type].$log->filename;- }- }- if (! empty($log->accept_signature)) {- $paths[] = FileStorage::Signatures->privateStorageKey().$log->accept_signature;- }- }-- foreach (array_unique($paths) as $path) {- $this->tryUnlink($path);}}