Skip to content

A free open source IT asset/license management system

Use chunk by ID in purge for larger data sets

snipeedc57459cb15
diff --git a/app/Console/Commands/Purge.php b/app/Console/Commands/Purge.php
index dd36e13282..bc265a82f9 100644
--- a/app/Console/Commands/Purge.php
+++ b/app/Console/Commands/Purge.php
@@ -187,16 +187,19 @@ class Purge extends Command
// Polymorphic action_log cleanup. Every model referenced by
// action_logs uses one of two column pairs. Users use target_*,
- // everything else uses item_*.
+ // everything else uses item_*. Chunked whereIn turns what used
+ // to be N*2 individual DELETEs into (N/1000)*2 bulk ones. Fixes
+ // the big-purge timeout + memory bomb on sites with hundreds
+ // of thousands of trashed rows.
$itemLogs = 0;
$targetLogs = 0;
- foreach ($ids as $id) {
+ foreach ($ids->chunk(1000) as $chunk) {
$itemQuery = DB::table('action_logs')
->where('item_type', $modelClass)
- ->where('item_id', $id);
+ ->whereIn('item_id', $chunk);
$targetQuery = DB::table('action_logs')
->where('target_type', $modelClass)
- ->where('target_id', $id);
+ ->whereIn('target_id', $chunk);
if ($dryRun) {
$itemLogs += $itemQuery->count();
$targetLogs += $targetQuery->count();
@@ -258,17 +261,20 @@ class Purge extends Command
Component::class => ['checkout_requests' => ['requestable_id', 'requestable_type', Component::class]],
AssetModel::class => ['checkout_requests' => ['requestable_id', 'requestable_type', AssetModel::class]],
];
+ // Chunked whereIn, same pattern as the action_log cleanup
+ // above. Replaces N*k individual DELETEs (k = number of
+ // child-table entries for this model) with (N/1000)*k bulk ones.
$childCounts = [];
if (array_key_exists($modelClass, $childTables)) {
foreach ($childTables[$modelClass] as $childTable => $foreignKey) {
$count = 0;
- foreach ($ids as $id) {
+ foreach ($ids->chunk(1000) as $chunk) {
$q = DB::table($childTable);
if (is_array($foreignKey)) {
[$idColumn, $typeColumn, $expectedType] = $foreignKey;
- $q->where($idColumn, $id)->where($typeColumn, $expectedType);
+ $q->whereIn($idColumn, $chunk)->where($typeColumn, $expectedType);
} else {
- $q->where($foreignKey, $id);
+ $q->whereIn($foreignKey, $chunk);
}
$count += $dryRun ? $q->count() : $q->delete();
}
@@ -394,43 +400,56 @@ class Purge extends Command
}
}
- // Chunk the IN list so a user with tens of thousands of
- // trashed rows doesn't blow MySQL's placeholder ceiling
- // on a single prepared statement. Collected into one $logs
- // collection so the dedupe + unlink step below stays reasonable.
- $logs = $ids->chunk(1000)->flatMap(fn ($chunk) => DB::table('action_logs')
- ->select('action_type', 'item_type', 'filename', 'accept_signature')
- ->where(function ($outer) use ($modelClass, $chunk) {
- $outer->where(function ($s) use ($modelClass, $chunk) {
- $s->where('item_type', $modelClass)->whereIn('item_id', $chunk);
- })->orWhere(function ($s) use ($modelClass, $chunk) {
- $s->where('target_type', $modelClass)->whereIn('target_id', $chunk);
+ // Chunk the parent ids AND stream the matching action_logs via
+ // chunkById so only one page of logs lives in memory at a time.
+ // The old shape collected every log row into a single Collection
+ // (via flatMap(...)->get()) and OOM'd a web request on sites
+ // with hundreds of thousands of rows per model. Each chunk's
+ // paths unlink immediately before the next chunk loads.
+ //
+ // $seenPaths dedupes across chunks so the same signature or
+ // EULA file (referenced by multiple logs) isn't unlinked twice.
+ $seenPaths = [];
+ foreach ($ids->chunk(1000) as $idChunk) {
+ DB::table('action_logs')
+ ->select('id', 'action_type', 'item_type', 'filename', 'accept_signature')
+ ->where(function ($outer) use ($modelClass, $idChunk) {
+ $outer->where(function ($s) use ($modelClass, $idChunk) {
+ $s->where('item_type', $modelClass)->whereIn('item_id', $idChunk);
+ })->orWhere(function ($s) use ($modelClass, $idChunk) {
+ $s->where('target_type', $modelClass)->whereIn('target_id', $idChunk);
+ });
+ })
+ ->orderBy('id')
+ ->chunkById(1000, function ($logs) use (&$seenPaths, $uploadRoots) {
+ foreach ($logs as $log) {
+ // Map action_type to disk path for the attached file.
+ // Mirrors Actionlog::uploads_file_path() but works
+ // off raw query-builder rows (no Eloquent).
+ if (! empty($log->filename)) {
+ if ($log->action_type === ActionType::Accepted->value || $log->action_type === ActionType::Declined->value) {
+ $path = FileStorage::EulaPdfs->privateStorageKey().$log->filename;
+ } elseif ($log->action_type === ActionType::Audit->value) {
+ $path = FileStorage::Audits->privateStorageKey().$log->filename;
+ } elseif ($log->item_type && isset($uploadRoots[$log->item_type])) {
+ $path = $uploadRoots[$log->item_type].$log->filename;
+ } else {
+ $path = null;
+ }
+ if ($path !== null && ! isset($seenPaths[$path])) {
+ $seenPaths[$path] = true;
+ $this->tryUnlink($path);
+ }
+ }
+ if (! empty($log->accept_signature)) {
+ $sigPath = FileStorage::Signatures->privateStorageKey().$log->accept_signature;
+ if (! isset($seenPaths[$sigPath])) {
+ $seenPaths[$sigPath] = true;
+ $this->tryUnlink($sigPath);
+ }
+ }
+ }
});
- })
- ->get());
-
- $paths = [];
- foreach ($logs as $log) {
- // Map action_type to disk path for the attached file, or
- // skip if the log carries no attachment we know how to route.
- // Mirrors Actionlog::uploads_file_path() but works off raw
- // query-builder rows (no Eloquent).
- if (! empty($log->filename)) {
- if ($log->action_type === ActionType::Accepted->value || $log->action_type === ActionType::Declined->value) {
- $paths[] = FileStorage::EulaPdfs->privateStorageKey().$log->filename;
- } elseif ($log->action_type === ActionType::Audit->value) {
- $paths[] = FileStorage::Audits->privateStorageKey().$log->filename;
- } elseif ($log->item_type && isset($uploadRoots[$log->item_type])) {
- $paths[] = $uploadRoots[$log->item_type].$log->filename;
- }
- }
- if (! empty($log->accept_signature)) {
- $paths[] = FileStorage::Signatures->privateStorageKey().$log->accept_signature;
- }
- }
-
- foreach (array_unique($paths) as $path) {
- $this->tryUnlink($path);
}
}