Skip to content

Beautiful Open-Source Form Builder

Fix traffic breakdown data loss during hourly cleanup (#1316)

Julien Nahumc851f9fb3c58
diff --git a/api/app/Console/Commands/CleanDatabase.php b/api/app/Console/Commands/CleanDatabase.php
index d624c85a8..06b048146 100644
--- a/api/app/Console/Commands/CleanDatabase.php
+++ b/api/app/Console/Commands/CleanDatabase.php
@@ -4,11 +4,15 @@ namespace App\Console\Commands;
use App\Models\Forms\FormStatistic;
use App\Models\Forms\FormView;
+use App\Service\Forms\Analytics\FormViewStatistics;
use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
class CleanDatabase extends Command
{
+ private const VIEW_BATCH_SIZE = 1000;
+
/**
* The name and signature of the console command.
*
@@ -43,52 +47,63 @@ class CleanDatabase extends Command
private function cleanFormStatistics()
{
$this->line('Aggregating form views...');
+ $statistics = app(FormViewStatistics::class);
$now = now();
- $finalData = [];
-
- // Form Views
- FormView::select('form_id', DB::raw('DATE(created_at) as date'), DB::raw('count(*) as views'))
- ->where('created_at', '<', $now)
- ->orderBy('date')
- ->groupBy('form_id', 'date')
- ->get()->each(function ($row) use (&$finalData) {
- $finalData[$row->form_id.'-'.$row->date] = [
- 'form_id' => $row->form_id,
- 'date' => $row->date,
- 'data' => [
- 'views' => $row->views,
- 'submissions' => 0,
- ],
- ];
- });
-
- if ($finalData) {
- $this->line('Storing aggregated data...');
- $created = 0;
- $updated = 0;
- // Insert into Form Statistic
- foreach ($finalData as $row) {
- $found = FormStatistic::where([['form_id', $row['form_id']], ['date', $row['date']]])->first();
- if ($found !== null) { // If found update
- $newData = $found->data;
- $newData['views'] = $newData['views'] + $row['data']['views'];
- $newData['submissions'] = 0;
- $found->update(['data' => $newData]);
- $updated++;
- } else { // Otherwise create new
- FormStatistic::create($row);
- $created++;
+ $maxId = FormView::max('id') ?? 0;
+
+ do {
+ $batch = DB::transaction(function () use ($statistics, $now, $maxId) {
+ $ids = DB::table('form_views')->where('created_at', '<', $now)->where('id', '<=', $maxId)
+ ->orderBy('id')->limit(self::VIEW_BATCH_SIZE)->lockForUpdate()->pluck('id');
+ if ($ids->isEmpty()) {
+ return null;
}
- }
- $this->line($created.' form statistics records created.');
- $this->line($updated.' form statistics records updated.');
+ $views = DB::table('form_views')->whereIn('id', $ids);
+ $finalData = [];
+
+ $statistics->groupedViews((clone $views)->select('form_id')->selectRaw('DATE(created_at) as date'))
+ ->groupBy('form_id')->groupByRaw('DATE(created_at)')->orderBy('form_id')->orderBy('date')
+ ->get()->each(function ($row) use (&$finalData, $statistics) {
+ $key = $row->form_id . '-' . $row->date;
+ $finalData[$key] ??= [
+ 'form_id' => $row->form_id,
+ 'date' => $row->date,
+ 'data' => ['views' => 0, 'submissions' => 0, 'meta_stats' => $statistics->emptyMetadata()],
+ ];
+ $finalData[$key]['data']['views'] += (int) $row->count;
+ $finalData[$key]['data']['meta_stats'] = $statistics->mergeMetadata(
+ $finalData[$key]['data']['meta_stats'],
+ $statistics->rowMetadata($row)
+ );
+ });
+
+ // Serialize daily-row creation even when concurrent batches have disjoint views.
+ $formIds = array_unique(array_column($finalData, 'form_id'));
+ DB::table('forms')->whereIn('id', $formIds)->orderBy('id')->lockForUpdate()->get(['id']);
- // Delete Form Views those are migrated
- $formViewRemovedCount = FormView::where('created_at', '<', $now)->delete();
- $this->line($formViewRemovedCount.' form views records deleted.');
- } else {
- $this->line('No aggregate to store.');
- }
+ foreach ($finalData as $row) {
+ $found = FormStatistic::where('form_id', $row['form_id'])->where('date', $row['date'])->lockForUpdate()->first();
+ if ($found !== null) {
+ $data = $found->data;
+ $data['views'] = ($data['views'] ?? 0) + $row['data']['views'];
+ $data['meta_stats'] = $statistics->mergeMetadata($data['meta_stats'] ?? [], $row['data']['meta_stats']);
+ $found->update(['data' => $data]);
+ } else {
+ FormStatistic::create($row);
+ }
+ }
+
+ // Delete only the exact locked rows whose counts have been stored.
+ return ['removed' => $views->delete(), 'form_ids' => $formIds];
+ }, 3);
+
+ if ($batch !== null) {
+ foreach ($batch['form_ids'] as $formId) {
+ Cache::forget('form_stats_metadata_v2_' . $formId);
+ }
+ $this->line($batch['removed'] . ' form views records aggregated and deleted.');
+ }
+ } while ($batch !== null);
}
}
diff --git a/api/app/Console/Kernel.php b/api/app/Console/Kernel.php
index 807f7c5a8..022b48325 100644
--- a/api/app/Console/Kernel.php
+++ b/api/app/Console/Kernel.php
@@ -23,7 +23,7 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
- $schedule->command('forms:database-cleanup')->hourly();
+ $schedule->command('forms:database-cleanup')->hourly()->withoutOverlapping();
$schedule->command('forms:purge-expired-submissions')->hourly()->withoutOverlapping();
$schedule->command('forms:retry-pending-submission-file-deletions')->everyFifteenMinutes()->withoutOverlapping();
$schedule->command('forms:integration-events-cleanup')->daily();
diff --git a/api/app/Http/Controllers/Forms/FormStatsController.php b/api/app/Http/Controllers/Forms/FormStatsController.php
index 02bd8e41b..f5759594f 100644
--- a/api/app/Http/Controllers/Forms/FormStatsController.php
+++ b/api/app/Http/Controllers/Forms/FormStatsController.php
@@ -10,7 +10,7 @@ use Carbon\CarbonPeriod;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
use App\Models\Forms\Form;
-use Illuminate\Support\Facades\DB;
+use App\Service\Forms\Analytics\FormViewStatistics;
class FormStatsController extends Controller
{
@@ -56,61 +56,8 @@ class FormStatsController extends Controller
return $submissionsWithDuration > 0 ? round($totalDuration / $submissionsWithDuration) : null;
});
- // Aggregate metadata from form views using single database query
- $metaStats = Cache::remember('form_stats_metadata_' . $form->id, 3600, function () use ($form) {
- $metadataFields = [
- 'source',
- 'device',
- 'country',
- 'browser',
- 'os'
- ];
-
- // Determine JSON extraction syntax based on database type
- $isMySQL = config('database.default') === 'mysql';
- $jsonExtractPattern = $isMySQL
- // Normalize empty string and literal 'null' to 'Unknown' in MySQL
- ? "COALESCE(NULLIF(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(meta, '$.%s')), ''), 'null'), 'Unknown')"
- : "COALESCE(meta->>'%s', 'Unknown')";
-
- // Build query with safe field binding
- $query = DB::table('form_views')->where('form_id', $form->id);
-
- $selectFields = [];
- $groupByFields = [];
-
- foreach ($metadataFields as $field) {
- $extractExpression = sprintf($jsonExtractPattern, $field);
- $selectFields[] = DB::raw("$extractExpression as $field");
- $groupByFields[] = DB::raw($extractExpression);
- }
-
- $results = $query
- ->select(array_merge($selectFields, [DB::raw('COUNT(*) as count')]))
- ->groupBy($groupByFields)
- ->orderBy('count', 'desc')
- ->get();
-
- // Initialize stats arrays
- $stats = [];
- foreach ($metadataFields as $field) {
- $stats[$field] = [];
- }
-
- // Aggregate results into stats arrays
- foreach ($results as $row) {
- foreach ($metadataFields as $field) {
- $value = $row->$field;
- $stats[$field][$value] = ($stats[$field][$value] ?? 0) + $row->count;
- }
- }
-
- // Sort each field by count (descending)
- foreach ($metadataFields as $field) {
- arsort($stats[$field]);
- }
-
- return $stats;
+ $metaStats = Cache::remember('form_stats_metadata_v2_' . $form->id, 300, function () use ($form) {
+ return app(FormViewStatistics::class)->metadata($form);
});
return [
diff --git a/api/app/Service/Forms/Analytics/FormViewStatistics.php b/api/app/Service/Forms/Analytics/FormViewStatistics.php
new file mode 100644
index 000000000..6b4acddba
--- /dev/null
+++ b/api/app/Service/Forms/Analytics/FormViewStatistics.php
@@ -0,0 +1,78 @@
+<?php
+
+namespace App\Service\Forms\Analytics;
+
+use App\Models\Forms\Form;
+use Illuminate\Database\Query\Builder;
+use Illuminate\Support\Facades\DB;
+
+class FormViewStatistics
+{
+ private const FIELDS = ['source', 'device', 'country', 'browser', 'os'];
+
+ public function emptyMetadata(): array
+ {
+ return array_fill_keys(self::FIELDS, []);
+ }
+
+ public function mergeMetadata(array $stats, array $counts): array
+ {
+ foreach (self::FIELDS as $field) {
+ foreach ($counts[$field] ?? [] as $value => $count) {
+ $stats[$field][$value] = ($stats[$field][$value] ?? 0) + $count;
+ }
+ }
+
+ return $stats;
+ }
+
+ public function rowMetadata(object $row): array
+ {
+ $stats = $this->emptyMetadata();
+ foreach (self::FIELDS as $field) {
+ $stats[$field][$row->$field] = (int) $row->count;
+ }
+
+ return $stats;
+ }
+
+ public function groupedViews(Builder $query): Builder
+ {
+ $pattern = $query->getConnection()->getDriverName() === 'mysql'
+ ? "COALESCE(NULLIF(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(meta, '$.%s')), ''), 'null'), 'Unknown')"
+ : "COALESCE(NULLIF(NULLIF(meta->>'%s', ''), 'null'), 'Unknown')";
+
+ foreach (self::FIELDS as $field) {
+ $expression = sprintf($pattern, $field);
+ $query->selectRaw("$expression as $field")->groupByRaw($expression);
+ }
+
+ return $query->selectRaw('COUNT(*) as count');
+ }
+
+ public function metadata(Form $form): array
+ {
+ $views = $this->groupedViews(DB::table('form_views')->where('form_id', $form->id))
+ ->selectRaw('NULL as data');
+ $history = DB::table('form_statistics')->where('form_id', $form->id);
+ foreach (self::FIELDS as $field) {
+ $history->selectRaw("NULL as $field");
+ }
+ $history->selectRaw('0 as count')->addSelect('data');
+
+ // One statement keeps both sources on the same snapshot during cleanup.
+ $stats = $this->emptyMetadata();
+ foreach ($views->unionAll($history)->cursor() as $row) {
+ $counts = $row->data === null
+ ? $this->rowMetadata($row)
+ : (json_decode($row->data, true)['meta_stats'] ?? []);
+ $stats = $this->mergeMetadata($stats, $counts);
+ }
+
+ foreach (self::FIELDS as $field) {
+ arsort($stats[$field]);
+ }
+
+ return $stats;
+ }
+}
diff --git a/api/tests/Feature/Forms/CleanDatabaseTest.php b/api/tests/Feature/Forms/CleanDatabaseTest.php
index c3ca3e9bb..d473f672e 100644
--- a/api/tests/Feature/Forms/CleanDatabaseTest.php
+++ b/api/tests/Feature/Forms/CleanDatabaseTest.php
@@ -42,7 +42,13 @@ it('check form statistic for views & submissions counts', function () {
expect($form->submissions()->count())->toBe(16);
expect(count($statistics))->toBe(2); // 1 per day for 2 different dates
expect($statistics[0]['date'])->toBe(now()->subDays(2)->toDateString());
- expect($statistics[0]['data'])->toBe(['views' => 1, 'submissions' => 0]);
+ expect($statistics[0]['data']['views'])->toBe(1);
+ expect($statistics[0]['data']['submissions'])->toBe(0);
expect($statistics[1]['date'])->toBe(now()->subDay()->toDateString());
- expect($statistics[1]['data'])->toBe(['views' => 10, 'submissions' => 0]);
+ expect($statistics[1]['data']['views'])->toBe(10);
+ expect($statistics[1]['data']['submissions'])->toBe(0);
+ foreach (['source', 'device', 'country', 'browser', 'os'] as $field) {
+ expect($statistics[0]['data']['meta_stats'][$field])->toBe(['Unknown' => 1]);
+ expect($statistics[1]['data']['meta_stats'][$field])->toBe(['Unknown' => 10]);
+ }
});
diff --git a/api/tests/Feature/Forms/FormStatTest.php b/api/tests/Feature/Forms/FormStatTest.php
index 3c0f816c1..406598d5e 100644
--- a/api/tests/Feature/Forms/FormStatTest.php
+++ b/api/tests/Feature/Forms/FormStatTest.php
@@ -68,10 +68,6 @@ it('check formstat chart data', function () {
it('checks form stats details', function () {
- if (config('database.default') === 'sqlite') {
- $this->markTestSkipped('Skipped on sqlite because the stats query uses PostgreSQL JSON casting syntax.');
- }
-
$user = $this->actingAsProUser();
$workspace = $this->createUserWorkspace($user);
$form = $this->createForm($user, $workspace, []);
diff --git a/api/tests/Feature/Forms/FormTrafficStatisticsTest.php b/api/tests/Feature/Forms/FormTrafficStatisticsTest.php
new file mode 100644
index 000000000..b748afab1
--- /dev/null
+++ b/api/tests/Feature/Forms/FormTrafficStatisticsTest.php
@@ -0,0 +1,175 @@
+<?php
+
+use App\Models\Forms\FormStatistic;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\Cache;
+
+beforeEach(function () {
+ $this->travelTo(now()->startOfDay()->addHours(12));
+ $user = $this->actingAsProUser();
+ $this->workspace = $this->createUserWorkspace($user);
+ $this->form = $this->createForm($user, $this->workspace);
+ $this->statsUrl = route('open.workspaces.form.stats-details', [$this->workspace, $this->form]);
+});
+
+it('preserves traffic breakdown across repeated daily cleanups and combines recent views', function () {
+ $meta = ['source' => 'Google', 'device' => 'Desktop', 'country' => 'FR', 'browser' => 'Firefox', 'os' => 'Linux'];
+ foreach ([now()->subDay(), now()->subDay(), now()->subMinute()] as $date) {
+ $view = $this->form->views()->create(['meta' => $meta]);
+ $view->forceFill(['created_at' => $date])->save();
+ }
+ $this->getJson($this->statsUrl)->assertSuccessful()->assertJsonPath('meta_stats.source.Google', 3);
+
+ Artisan::call('forms:database-cleanup');
+ expect($this->form->views()->count())->toBe(0);
+ expect($this->form->statistics()->count())->toBe(2);
+ Cache::forget('form_stats_metadata_v2_' . $this->form->id);
+ foreach ($meta as $field => $value) {
+ $this->getJson($this->statsUrl)->assertSuccessful()->assertJsonPath('meta_stats.' . $field . '.' . $value, 3);
+ }
+
+ $view = $this->form->views()->create(['meta' => $meta]);
+ $view->forceFill(['created_at' => now()->subSecond()])->save();
+ Artisan::call('forms:database-cleanup');
+ // The cleanup invalidates the previously cached result.
+ $this->getJson($this->statsUrl)->assertSuccessful()->assertJsonPath('meta_stats.source.Google', 4);
+ Artisan::call('forms:database-cleanup');
+ expect($this->form->statistics()->get()->sum(fn ($row) => $row->data['views']))->toBe(4);
+
+ $this->form->views()->create(['meta' => $meta]);
+ $this->form->views()->create(['meta' => ['source' => 'Direct']]);
+ Cache::forget('form_stats_metadata_v2_' . $this->form->id);
+ $response = $this->getJson($this->statsUrl)->assertSuccessful()
+ ->assertJsonPath('meta_stats.source.Google', 5)
+ ->assertJsonPath('meta_stats.source.Direct', 1)
+ ->assertJsonPath('meta_stats.device.Unknown', 1);
+ expect(array_keys($response->json('meta_stats.source')))->toBe(['Google', 'Direct']);
+});
+
+it('normalizes missing metadata and leaves legacy statistics and other forms isolated', function () {
+ $this->form->statistics()->create([
+ 'date' => now()->subDay()->toDateString(),
+ 'data' => ['views' => 50, 'submissions' => 7, 'custom' => 'preserved'],
+ ]);
+ foreach ([null, [], ['source' => null], ['source' => ''], ['source' => 'null']] as $meta) {
+ $view = $this->form->views()->create(['meta' => $meta]);
+ $view->forceFill(['created_at' => now()->subDay()])->save();
+ }
+ $otherForm = $this->createForm($this->form->creator, $this->workspace);
+ $view = $otherForm->views()->create(['meta' => ['source' => 'Other']]);
+ $view->forceFill(['created_at' => now()->subDay()])->save();
+
+ Artisan::call('forms:database-cleanup');
+ $data = $this->form->statistics()->first()->data;
+ expect($data['views'])->toBe(55);
+ expect($data['submissions'])->toBe(7);
+ expect($data['custom'])->toBe('preserved');
+ $response = $this->getJson($this->statsUrl)->assertSuccessful();
+ foreach (['source', 'device', 'country', 'browser', 'os'] as $field) {
+ expect($response->json('meta_stats.' . $field))->toBe(['Unknown' => 5]);
+ }
+});
+
+it('keeps views at the cutoff and rolls back aggregation if storing statistics fails', function () {
+ $oldView = $this->form->views()->create(['meta' => ['source' => 'Google']]);
+ $oldView->forceFill(['created_at' => now()->subDay()])->save();
+ $olderView = $this->form->views()->create(['meta' => ['source' => 'Google']]);
+ $olderView->forceFill(['created_at' => now()->subDays(2)])->save();
+ $recentView = $this->form->views()->create(['meta' => ['source' => 'Direct']]);
+
+ $writes = 0;
+ FormStatistic::creating(function () use (&$writes) {
+ if (++$writes === 2) {
+ throw new RuntimeException('Storage failed');
+ }
+ });
+ try {
+ expect(fn () => Artisan::call('forms:database-cleanup'))->toThrow(RuntimeException::class, 'Storage failed');
+ expect($this->form->views()->count())->toBe(3);
+ expect($this->form->statistics()->count())->toBe(0);
+ } finally {
+ FormStatistic::flushEventListeners();
+ }
+
+ Artisan::call('forms:database-cleanup');
+ expect($this->form->views()->pluck('id')->all())->toBe([$recentView->id]);
+ $this->getJson($this->statsUrl)->assertSuccessful()
+ ->assertJsonPath('meta_stats.source.Google', 2)
+ ->assertJsonPath('meta_stats.source.Direct', 1);
+});
+
+it('does not delete a backdated view inserted during aggregation', function () {
+ $view = $this->form->views()->create(['meta' => ['source' => 'Google']]);
+ $view->forceFill(['created_at' => now()->subDay()])->save();
+ FormStatistic::created(function () {
+ $view = $this->form->views()->create(['meta' => ['source' => 'Direct']]);
+ $view->forceFill(['created_at' => now()->subDay()])->save();
+ });
+ try {
+ Artisan::call('forms:database-cleanup');
+ } finally {
+ FormStatistic::flushEventListeners();
+ }
+
+ expect($this->form->views()->count())->toBe(1);
+ $this->getJson($this->statsUrl)->assertSuccessful()
+ ->assertJsonPath('meta_stats.source.Google', 1)
+ ->assertJsonPath('meta_stats.source.Direct', 1);
+});
+
+it('returns empty breakdowns for forms without metadata history or views', function () {
+ $this->form->statistics()->create(['date' => now()->toDateString(), 'data' => ['views' => 50]]);
+ $response = $this->getJson($this->statsUrl)->assertSuccessful();
+ foreach (['source', 'device', 'country', 'browser', 'os'] as $field) {
+ expect($response->json('meta_stats.' . $field))->toBe([]);
+ }
+});
+
+it('preserves an earlier view that becomes visible after aggregation', function () {
+ $pendingView = $this->form->views()->create(['meta' => ['source' => 'Direct']]);
+ $this->form->views()->whereKey($pendingView->id)->delete();
+ $view = $this->form->views()->create(['meta' => ['source' => 'Google']]);
+ $view->forceFill(['created_at' => now()->subDay()])->save();
+
+ // Simulate a previously allocated ID committing after the aggregate read.
+ FormStatistic::created(function () use ($pendingView) {
+ $this->form->views()->insert([
+ 'id' => $pendingView->id,
+ 'form_id' => $this->form->id,
+ 'meta' => json_encode(['source' => 'Direct']),
+ 'created_at' => now()->subDay(),
+ 'updated_at' => now()->subDay(),
+ ]);
+ });
+ try {
+ Artisan::call('forms:database-cleanup');
+ } finally {
+ FormStatistic::flushEventListeners();
+ }
+
+ expect($this->form->views()->count())->toBe(0);
+ expect($this->form->statistics()->first()->data['views'])->toBe(2);
+ $this->getJson($this->statsUrl)->assertSuccessful()
+ ->assertJsonPath('meta_stats.source.Google', 1)
+ ->assertJsonPath('meta_stats.source.Direct', 1);
+});
+
+it('processes multiple bounded batches without losing or duplicating counts', function () {
+ $rows = array_fill(0, 1005, [
+ 'form_id' => $this->form->id,
+ 'meta' => json_encode(['source' => 'Google']),
+ 'created_at' => now()->subDay(),
+ 'updated_at' => now()->subDay(),
+ ]);
+ $this->form->views()->insert($rows);
+
+ Artisan::call('forms:database-cleanup');
+ Artisan::call('forms:database-cleanup');
+
+ expect($this->form->views()->count())->toBe(0);
+ expect($this->form->statistics()->count())->toBe(1);
+ expect($this->form->statistics()->first()->data['views'])->toBe(1005);
+ $this->getJson($this->statsUrl)->assertSuccessful()
+ ->assertJsonPath('meta_stats.source.Google', 1005)
+ ->assertJsonPath('meta_stats.device.Unknown', 1005);
+});