Beautiful Open-Source Form Builder
Fix traffic breakdown data loss during hourly cleanup (#1316)
Julien Nahumc851f9fb3c58
diff --git a/api/app/Console/Commands/CleanDatabase.php b/api/app/Console/Commands/CleanDatabase.phpindex d624c85a8..06b048146 100644--- a/api/app/Console/Commands/CleanDatabase.php+++ b/api/app/Console/Commands/CleanDatabase.php@@ -4,11 +4,15 @@ namespace App\Console\Commands;use App\Models\Forms\FormStatistic;use App\Models\Forms\FormView;+use App\Service\Forms\Analytics\FormViewStatistics;use Illuminate\Console\Command;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\DB;class CleanDatabase extends Command{+ private const VIEW_BATCH_SIZE = 1000;+/*** The name and signature of the console command.*@@ -43,52 +47,63 @@ class CleanDatabase extends Commandprivate function cleanFormStatistics(){$this->line('Aggregating form views...');+ $statistics = app(FormViewStatistics::class);$now = now();- $finalData = [];-- // Form Views- FormView::select('form_id', DB::raw('DATE(created_at) as date'), DB::raw('count(*) as views'))- ->where('created_at', '<', $now)- ->orderBy('date')- ->groupBy('form_id', 'date')- ->get()->each(function ($row) use (&$finalData) {- $finalData[$row->form_id.'-'.$row->date] = [- 'form_id' => $row->form_id,- 'date' => $row->date,- 'data' => [- 'views' => $row->views,- 'submissions' => 0,- ],- ];- });-- if ($finalData) {- $this->line('Storing aggregated data...');- $created = 0;- $updated = 0;- // Insert into Form Statistic- foreach ($finalData as $row) {- $found = FormStatistic::where([['form_id', $row['form_id']], ['date', $row['date']]])->first();- if ($found !== null) { // If found update- $newData = $found->data;- $newData['views'] = $newData['views'] + $row['data']['views'];- $newData['submissions'] = 0;- $found->update(['data' => $newData]);- $updated++;- } else { // Otherwise create new- FormStatistic::create($row);- $created++;+ $maxId = FormView::max('id') ?? 0;++ do {+ $batch = DB::transaction(function () use ($statistics, $now, $maxId) {+ $ids = DB::table('form_views')->where('created_at', '<', $now)->where('id', '<=', $maxId)+ ->orderBy('id')->limit(self::VIEW_BATCH_SIZE)->lockForUpdate()->pluck('id');+ if ($ids->isEmpty()) {+ return null;}- }- $this->line($created.' form statistics records created.');- $this->line($updated.' form statistics records updated.');+ $views = DB::table('form_views')->whereIn('id', $ids);+ $finalData = [];++ $statistics->groupedViews((clone $views)->select('form_id')->selectRaw('DATE(created_at) as date'))+ ->groupBy('form_id')->groupByRaw('DATE(created_at)')->orderBy('form_id')->orderBy('date')+ ->get()->each(function ($row) use (&$finalData, $statistics) {+ $key = $row->form_id . '-' . $row->date;+ $finalData[$key] ??= [+ 'form_id' => $row->form_id,+ 'date' => $row->date,+ 'data' => ['views' => 0, 'submissions' => 0, 'meta_stats' => $statistics->emptyMetadata()],+ ];+ $finalData[$key]['data']['views'] += (int) $row->count;+ $finalData[$key]['data']['meta_stats'] = $statistics->mergeMetadata(+ $finalData[$key]['data']['meta_stats'],+ $statistics->rowMetadata($row)+ );+ });++ // Serialize daily-row creation even when concurrent batches have disjoint views.+ $formIds = array_unique(array_column($finalData, 'form_id'));+ DB::table('forms')->whereIn('id', $formIds)->orderBy('id')->lockForUpdate()->get(['id']);- // Delete Form Views those are migrated- $formViewRemovedCount = FormView::where('created_at', '<', $now)->delete();- $this->line($formViewRemovedCount.' form views records deleted.');- } else {- $this->line('No aggregate to store.');- }+ foreach ($finalData as $row) {+ $found = FormStatistic::where('form_id', $row['form_id'])->where('date', $row['date'])->lockForUpdate()->first();+ if ($found !== null) {+ $data = $found->data;+ $data['views'] = ($data['views'] ?? 0) + $row['data']['views'];+ $data['meta_stats'] = $statistics->mergeMetadata($data['meta_stats'] ?? [], $row['data']['meta_stats']);+ $found->update(['data' => $data]);+ } else {+ FormStatistic::create($row);+ }+ }++ // Delete only the exact locked rows whose counts have been stored.+ return ['removed' => $views->delete(), 'form_ids' => $formIds];+ }, 3);++ if ($batch !== null) {+ foreach ($batch['form_ids'] as $formId) {+ Cache::forget('form_stats_metadata_v2_' . $formId);+ }+ $this->line($batch['removed'] . ' form views records aggregated and deleted.');+ }+ } while ($batch !== null);}}diff --git a/api/app/Console/Kernel.php b/api/app/Console/Kernel.phpindex 807f7c5a8..022b48325 100644--- a/api/app/Console/Kernel.php+++ b/api/app/Console/Kernel.php@@ -23,7 +23,7 @@ class Kernel extends ConsoleKernel*/protected function schedule(Schedule $schedule){- $schedule->command('forms:database-cleanup')->hourly();+ $schedule->command('forms:database-cleanup')->hourly()->withoutOverlapping();$schedule->command('forms:purge-expired-submissions')->hourly()->withoutOverlapping();$schedule->command('forms:retry-pending-submission-file-deletions')->everyFifteenMinutes()->withoutOverlapping();$schedule->command('forms:integration-events-cleanup')->daily();diff --git a/api/app/Http/Controllers/Forms/FormStatsController.php b/api/app/Http/Controllers/Forms/FormStatsController.phpindex 02bd8e41b..f5759594f 100644--- a/api/app/Http/Controllers/Forms/FormStatsController.php+++ b/api/app/Http/Controllers/Forms/FormStatsController.php@@ -10,7 +10,7 @@ use Carbon\CarbonPeriod;use Illuminate\Http\Request;use Illuminate\Support\Facades\Cache;use App\Models\Forms\Form;-use Illuminate\Support\Facades\DB;+use App\Service\Forms\Analytics\FormViewStatistics;class FormStatsController extends Controller{@@ -56,61 +56,8 @@ class FormStatsController extends Controllerreturn $submissionsWithDuration > 0 ? round($totalDuration / $submissionsWithDuration) : null;});- // Aggregate metadata from form views using single database query- $metaStats = Cache::remember('form_stats_metadata_' . $form->id, 3600, function () use ($form) {- $metadataFields = [- 'source',- 'device',- 'country',- 'browser',- 'os'- ];-- // Determine JSON extraction syntax based on database type- $isMySQL = config('database.default') === 'mysql';- $jsonExtractPattern = $isMySQL- // Normalize empty string and literal 'null' to 'Unknown' in MySQL- ? "COALESCE(NULLIF(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(meta, '$.%s')), ''), 'null'), 'Unknown')"- : "COALESCE(meta->>'%s', 'Unknown')";-- // Build query with safe field binding- $query = DB::table('form_views')->where('form_id', $form->id);-- $selectFields = [];- $groupByFields = [];-- foreach ($metadataFields as $field) {- $extractExpression = sprintf($jsonExtractPattern, $field);- $selectFields[] = DB::raw("$extractExpression as $field");- $groupByFields[] = DB::raw($extractExpression);- }-- $results = $query- ->select(array_merge($selectFields, [DB::raw('COUNT(*) as count')]))- ->groupBy($groupByFields)- ->orderBy('count', 'desc')- ->get();-- // Initialize stats arrays- $stats = [];- foreach ($metadataFields as $field) {- $stats[$field] = [];- }-- // Aggregate results into stats arrays- foreach ($results as $row) {- foreach ($metadataFields as $field) {- $value = $row->$field;- $stats[$field][$value] = ($stats[$field][$value] ?? 0) + $row->count;- }- }-- // Sort each field by count (descending)- foreach ($metadataFields as $field) {- arsort($stats[$field]);- }-- return $stats;+ $metaStats = Cache::remember('form_stats_metadata_v2_' . $form->id, 300, function () use ($form) {+ return app(FormViewStatistics::class)->metadata($form);});return [diff --git a/api/app/Service/Forms/Analytics/FormViewStatistics.php b/api/app/Service/Forms/Analytics/FormViewStatistics.phpnew file mode 100644index 000000000..6b4acddba--- /dev/null+++ b/api/app/Service/Forms/Analytics/FormViewStatistics.php@@ -0,0 +1,78 @@+<?php++namespace App\Service\Forms\Analytics;++use App\Models\Forms\Form;+use Illuminate\Database\Query\Builder;+use Illuminate\Support\Facades\DB;++class FormViewStatistics+{+ private const FIELDS = ['source', 'device', 'country', 'browser', 'os'];++ public function emptyMetadata(): array+ {+ return array_fill_keys(self::FIELDS, []);+ }++ public function mergeMetadata(array $stats, array $counts): array+ {+ foreach (self::FIELDS as $field) {+ foreach ($counts[$field] ?? [] as $value => $count) {+ $stats[$field][$value] = ($stats[$field][$value] ?? 0) + $count;+ }+ }++ return $stats;+ }++ public function rowMetadata(object $row): array+ {+ $stats = $this->emptyMetadata();+ foreach (self::FIELDS as $field) {+ $stats[$field][$row->$field] = (int) $row->count;+ }++ return $stats;+ }++ public function groupedViews(Builder $query): Builder+ {+ $pattern = $query->getConnection()->getDriverName() === 'mysql'+ ? "COALESCE(NULLIF(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(meta, '$.%s')), ''), 'null'), 'Unknown')"+ : "COALESCE(NULLIF(NULLIF(meta->>'%s', ''), 'null'), 'Unknown')";++ foreach (self::FIELDS as $field) {+ $expression = sprintf($pattern, $field);+ $query->selectRaw("$expression as $field")->groupByRaw($expression);+ }++ return $query->selectRaw('COUNT(*) as count');+ }++ public function metadata(Form $form): array+ {+ $views = $this->groupedViews(DB::table('form_views')->where('form_id', $form->id))+ ->selectRaw('NULL as data');+ $history = DB::table('form_statistics')->where('form_id', $form->id);+ foreach (self::FIELDS as $field) {+ $history->selectRaw("NULL as $field");+ }+ $history->selectRaw('0 as count')->addSelect('data');++ // One statement keeps both sources on the same snapshot during cleanup.+ $stats = $this->emptyMetadata();+ foreach ($views->unionAll($history)->cursor() as $row) {+ $counts = $row->data === null+ ? $this->rowMetadata($row)+ : (json_decode($row->data, true)['meta_stats'] ?? []);+ $stats = $this->mergeMetadata($stats, $counts);+ }++ foreach (self::FIELDS as $field) {+ arsort($stats[$field]);+ }++ return $stats;+ }+}diff --git a/api/tests/Feature/Forms/CleanDatabaseTest.php b/api/tests/Feature/Forms/CleanDatabaseTest.phpindex c3ca3e9bb..d473f672e 100644--- a/api/tests/Feature/Forms/CleanDatabaseTest.php+++ b/api/tests/Feature/Forms/CleanDatabaseTest.php@@ -42,7 +42,13 @@ it('check form statistic for views & submissions counts', function () {expect($form->submissions()->count())->toBe(16);expect(count($statistics))->toBe(2); // 1 per day for 2 different datesexpect($statistics[0]['date'])->toBe(now()->subDays(2)->toDateString());- expect($statistics[0]['data'])->toBe(['views' => 1, 'submissions' => 0]);+ expect($statistics[0]['data']['views'])->toBe(1);+ expect($statistics[0]['data']['submissions'])->toBe(0);expect($statistics[1]['date'])->toBe(now()->subDay()->toDateString());- expect($statistics[1]['data'])->toBe(['views' => 10, 'submissions' => 0]);+ expect($statistics[1]['data']['views'])->toBe(10);+ expect($statistics[1]['data']['submissions'])->toBe(0);+ foreach (['source', 'device', 'country', 'browser', 'os'] as $field) {+ expect($statistics[0]['data']['meta_stats'][$field])->toBe(['Unknown' => 1]);+ expect($statistics[1]['data']['meta_stats'][$field])->toBe(['Unknown' => 10]);+ }});diff --git a/api/tests/Feature/Forms/FormStatTest.php b/api/tests/Feature/Forms/FormStatTest.phpindex 3c0f816c1..406598d5e 100644--- a/api/tests/Feature/Forms/FormStatTest.php+++ b/api/tests/Feature/Forms/FormStatTest.php@@ -68,10 +68,6 @@ it('check formstat chart data', function () {it('checks form stats details', function () {- if (config('database.default') === 'sqlite') {- $this->markTestSkipped('Skipped on sqlite because the stats query uses PostgreSQL JSON casting syntax.');- }-$user = $this->actingAsProUser();$workspace = $this->createUserWorkspace($user);$form = $this->createForm($user, $workspace, []);diff --git a/api/tests/Feature/Forms/FormTrafficStatisticsTest.php b/api/tests/Feature/Forms/FormTrafficStatisticsTest.phpnew file mode 100644index 000000000..b748afab1--- /dev/null+++ b/api/tests/Feature/Forms/FormTrafficStatisticsTest.php@@ -0,0 +1,175 @@+<?php++use App\Models\Forms\FormStatistic;+use Illuminate\Support\Facades\Artisan;+use Illuminate\Support\Facades\Cache;++beforeEach(function () {+ $this->travelTo(now()->startOfDay()->addHours(12));+ $user = $this->actingAsProUser();+ $this->workspace = $this->createUserWorkspace($user);+ $this->form = $this->createForm($user, $this->workspace);+ $this->statsUrl = route('open.workspaces.form.stats-details', [$this->workspace, $this->form]);+});++it('preserves traffic breakdown across repeated daily cleanups and combines recent views', function () {+ $meta = ['source' => 'Google', 'device' => 'Desktop', 'country' => 'FR', 'browser' => 'Firefox', 'os' => 'Linux'];+ foreach ([now()->subDay(), now()->subDay(), now()->subMinute()] as $date) {+ $view = $this->form->views()->create(['meta' => $meta]);+ $view->forceFill(['created_at' => $date])->save();+ }+ $this->getJson($this->statsUrl)->assertSuccessful()->assertJsonPath('meta_stats.source.Google', 3);++ Artisan::call('forms:database-cleanup');+ expect($this->form->views()->count())->toBe(0);+ expect($this->form->statistics()->count())->toBe(2);+ Cache::forget('form_stats_metadata_v2_' . $this->form->id);+ foreach ($meta as $field => $value) {+ $this->getJson($this->statsUrl)->assertSuccessful()->assertJsonPath('meta_stats.' . $field . '.' . $value, 3);+ }++ $view = $this->form->views()->create(['meta' => $meta]);+ $view->forceFill(['created_at' => now()->subSecond()])->save();+ Artisan::call('forms:database-cleanup');+ // The cleanup invalidates the previously cached result.+ $this->getJson($this->statsUrl)->assertSuccessful()->assertJsonPath('meta_stats.source.Google', 4);+ Artisan::call('forms:database-cleanup');+ expect($this->form->statistics()->get()->sum(fn ($row) => $row->data['views']))->toBe(4);++ $this->form->views()->create(['meta' => $meta]);+ $this->form->views()->create(['meta' => ['source' => 'Direct']]);+ Cache::forget('form_stats_metadata_v2_' . $this->form->id);+ $response = $this->getJson($this->statsUrl)->assertSuccessful()+ ->assertJsonPath('meta_stats.source.Google', 5)+ ->assertJsonPath('meta_stats.source.Direct', 1)+ ->assertJsonPath('meta_stats.device.Unknown', 1);+ expect(array_keys($response->json('meta_stats.source')))->toBe(['Google', 'Direct']);+});++it('normalizes missing metadata and leaves legacy statistics and other forms isolated', function () {+ $this->form->statistics()->create([+ 'date' => now()->subDay()->toDateString(),+ 'data' => ['views' => 50, 'submissions' => 7, 'custom' => 'preserved'],+ ]);+ foreach ([null, [], ['source' => null], ['source' => ''], ['source' => 'null']] as $meta) {+ $view = $this->form->views()->create(['meta' => $meta]);+ $view->forceFill(['created_at' => now()->subDay()])->save();+ }+ $otherForm = $this->createForm($this->form->creator, $this->workspace);+ $view = $otherForm->views()->create(['meta' => ['source' => 'Other']]);+ $view->forceFill(['created_at' => now()->subDay()])->save();++ Artisan::call('forms:database-cleanup');+ $data = $this->form->statistics()->first()->data;+ expect($data['views'])->toBe(55);+ expect($data['submissions'])->toBe(7);+ expect($data['custom'])->toBe('preserved');+ $response = $this->getJson($this->statsUrl)->assertSuccessful();+ foreach (['source', 'device', 'country', 'browser', 'os'] as $field) {+ expect($response->json('meta_stats.' . $field))->toBe(['Unknown' => 5]);+ }+});++it('keeps views at the cutoff and rolls back aggregation if storing statistics fails', function () {+ $oldView = $this->form->views()->create(['meta' => ['source' => 'Google']]);+ $oldView->forceFill(['created_at' => now()->subDay()])->save();+ $olderView = $this->form->views()->create(['meta' => ['source' => 'Google']]);+ $olderView->forceFill(['created_at' => now()->subDays(2)])->save();+ $recentView = $this->form->views()->create(['meta' => ['source' => 'Direct']]);++ $writes = 0;+ FormStatistic::creating(function () use (&$writes) {+ if (++$writes === 2) {+ throw new RuntimeException('Storage failed');+ }+ });+ try {+ expect(fn () => Artisan::call('forms:database-cleanup'))->toThrow(RuntimeException::class, 'Storage failed');+ expect($this->form->views()->count())->toBe(3);+ expect($this->form->statistics()->count())->toBe(0);+ } finally {+ FormStatistic::flushEventListeners();+ }++ Artisan::call('forms:database-cleanup');+ expect($this->form->views()->pluck('id')->all())->toBe([$recentView->id]);+ $this->getJson($this->statsUrl)->assertSuccessful()+ ->assertJsonPath('meta_stats.source.Google', 2)+ ->assertJsonPath('meta_stats.source.Direct', 1);+});++it('does not delete a backdated view inserted during aggregation', function () {+ $view = $this->form->views()->create(['meta' => ['source' => 'Google']]);+ $view->forceFill(['created_at' => now()->subDay()])->save();+ FormStatistic::created(function () {+ $view = $this->form->views()->create(['meta' => ['source' => 'Direct']]);+ $view->forceFill(['created_at' => now()->subDay()])->save();+ });+ try {+ Artisan::call('forms:database-cleanup');+ } finally {+ FormStatistic::flushEventListeners();+ }++ expect($this->form->views()->count())->toBe(1);+ $this->getJson($this->statsUrl)->assertSuccessful()+ ->assertJsonPath('meta_stats.source.Google', 1)+ ->assertJsonPath('meta_stats.source.Direct', 1);+});++it('returns empty breakdowns for forms without metadata history or views', function () {+ $this->form->statistics()->create(['date' => now()->toDateString(), 'data' => ['views' => 50]]);+ $response = $this->getJson($this->statsUrl)->assertSuccessful();+ foreach (['source', 'device', 'country', 'browser', 'os'] as $field) {+ expect($response->json('meta_stats.' . $field))->toBe([]);+ }+});++it('preserves an earlier view that becomes visible after aggregation', function () {+ $pendingView = $this->form->views()->create(['meta' => ['source' => 'Direct']]);+ $this->form->views()->whereKey($pendingView->id)->delete();+ $view = $this->form->views()->create(['meta' => ['source' => 'Google']]);+ $view->forceFill(['created_at' => now()->subDay()])->save();++ // Simulate a previously allocated ID committing after the aggregate read.+ FormStatistic::created(function () use ($pendingView) {+ $this->form->views()->insert([+ 'id' => $pendingView->id,+ 'form_id' => $this->form->id,+ 'meta' => json_encode(['source' => 'Direct']),+ 'created_at' => now()->subDay(),+ 'updated_at' => now()->subDay(),+ ]);+ });+ try {+ Artisan::call('forms:database-cleanup');+ } finally {+ FormStatistic::flushEventListeners();+ }++ expect($this->form->views()->count())->toBe(0);+ expect($this->form->statistics()->first()->data['views'])->toBe(2);+ $this->getJson($this->statsUrl)->assertSuccessful()+ ->assertJsonPath('meta_stats.source.Google', 1)+ ->assertJsonPath('meta_stats.source.Direct', 1);+});++it('processes multiple bounded batches without losing or duplicating counts', function () {+ $rows = array_fill(0, 1005, [+ 'form_id' => $this->form->id,+ 'meta' => json_encode(['source' => 'Google']),+ 'created_at' => now()->subDay(),+ 'updated_at' => now()->subDay(),+ ]);+ $this->form->views()->insert($rows);++ Artisan::call('forms:database-cleanup');+ Artisan::call('forms:database-cleanup');++ expect($this->form->views()->count())->toBe(0);+ expect($this->form->statistics()->count())->toBe(1);+ expect($this->form->statistics()->first()->data['views'])->toBe(1005);+ $this->getJson($this->statsUrl)->assertSuccessful()+ ->assertJsonPath('meta_stats.source.Google', 1005)+ ->assertJsonPath('meta_stats.device.Unknown', 1005);+});