A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Fixhancement: truncate large logs, improve auto-scroll (#11239)
shamoona0d3527d206e
diff --git a/src-ui/src/app/components/admin/logs/logs.component.html b/src-ui/src/app/components/admin/logs/logs.component.htmlindex 21df9f33b..bdc80583f 100644--- a/src-ui/src/app/components/admin/logs/logs.component.html+++ b/src-ui/src/app/components/admin/logs/logs.component.html@@ -3,9 +3,23 @@i18n-titleinfo="Review the log files for the application and for email checking."i18n-info>- <div class="form-check form-switch">- <input class="form-check-input" type="checkbox" role="switch" [(ngModel)]="autoRefreshEnabled">- <label class="form-check-label" for="autoRefreshSwitch" i18n>Auto refresh</label>+ <div class="input-group input-group-sm align-items-center">+ <div class="input-group input-group-sm me-3">+ <span class="input-group-text text-muted" i18n>Show</span>+ <input+ class="form-control"+ type="number"+ min="100"+ step="100"+ [(ngModel)]="limit"+ (ngModelChange)="onLimitChange($event)"+ style="width: 100px;">+ <span class="input-group-text text-muted" i18n>lines</span>+ </div>+ <div class="form-check form-switch mt-1">+ <input class="form-check-input" type="checkbox" role="switch" [(ngModel)]="autoRefreshEnabled">+ <label class="form-check-label" for="autoRefreshSwitch" i18n>Auto refresh</label>+ </div></div></pngx-page-header>diff --git a/src-ui/src/app/components/admin/logs/logs.component.spec.ts b/src-ui/src/app/components/admin/logs/logs.component.spec.tsindex 841fec44d..728916830 100644--- a/src-ui/src/app/components/admin/logs/logs.component.spec.ts+++ b/src-ui/src/app/components/admin/logs/logs.component.spec.ts@@ -67,7 +67,7 @@ describe('LogsComponent', () => {})it('should display logs with first log initially', () => {- expect(logSpy).toHaveBeenCalledWith('paperless')+ expect(logSpy).toHaveBeenCalledWith('paperless', 5000)fixture.detectChanges()expect(fixture.debugElement.nativeElement.textContent).toContain(paperless_logs[0]@@ -78,7 +78,7 @@ describe('LogsComponent', () => {fixture.debugElement.queryAll(By.directive(NgbNavLink))[1].nativeElement.dispatchEvent(new MouseEvent('click'))- expect(logSpy).toHaveBeenCalledWith('mail')+ expect(logSpy).toHaveBeenCalledWith('mail', 5000)})it('should handle error with no logs', () => {@@ -101,4 +101,13 @@ describe('LogsComponent', () => {jest.advanceTimersByTime(6000)expect(reloadSpy).toHaveBeenCalledTimes(2)})++ it('should debounce limit changes before reloading logs', () => {+ const initialCalls = reloadSpy.mock.calls.length+ component.onLimitChange(6000)+ jest.advanceTimersByTime(299)+ expect(reloadSpy).toHaveBeenCalledTimes(initialCalls)+ jest.advanceTimersByTime(1)+ expect(reloadSpy).toHaveBeenCalledTimes(initialCalls + 1)+ })})diff --git a/src-ui/src/app/components/admin/logs/logs.component.ts b/src-ui/src/app/components/admin/logs/logs.component.tsindex 488f9db26..68b88265d 100644--- a/src-ui/src/app/components/admin/logs/logs.component.ts+++ b/src-ui/src/app/components/admin/logs/logs.component.ts@@ -13,7 +13,7 @@ import {} from '@angular/core'import { FormsModule, ReactiveFormsModule } from '@angular/forms'import { NgbNavModule } from '@ng-bootstrap/ng-bootstrap'-import { filter, takeUntil, timer } from 'rxjs'+import { Subject, debounceTime, filter, takeUntil, timer } from 'rxjs'import { LogService } from 'src/app/services/rest/log.service'import { PageHeaderComponent } from '../../common/page-header/page-header.component'import { LoadingComponentWithPermissions } from '../../loading-component/loading.component'@@ -47,9 +47,17 @@ export class LogsComponentpublic autoRefreshEnabled: boolean = true+ public limit: number = 5000++ private readonly limitChange$ = new Subject<number>()+@ViewChild('logContainer') logContainer: CdkVirtualScrollViewportngOnInit(): void {+ this.limitChange$+ .pipe(debounceTime(300), takeUntil(this.unsubscribeNotifier))+ .subscribe(() => this.reloadLogs())+this.logService.list().pipe(takeUntil(this.unsubscribeNotifier))@@ -75,16 +83,33 @@ export class LogsComponentsuper.ngOnDestroy()}+ onLimitChange(limit: number): void {+ this.limitChange$.next(limit)+ }+reloadLogs() {this.loading = truethis.logService- .get(this.activeLog)+ .get(this.activeLog, this.limit).pipe(takeUntil(this.unsubscribeNotifier)).subscribe({next: (result) => {- this.logs = this.parseLogsWithLevel(result)this.loading = false- this.scrollToBottom()+ const parsed = this.parseLogsWithLevel(result)+ const hasChanges =+ parsed.length !== this.logs.length ||+ parsed.some((log, idx) => {+ const current = this.logs[idx]+ return (+ !current ||+ current.message !== log.message ||+ current.level !== log.level+ )+ })+ if (hasChanges) {+ this.logs = parsed+ this.scrollToBottom()+ }},error: () => {this.logs = []diff --git a/src-ui/src/app/services/rest/log.service.spec.ts b/src-ui/src/app/services/rest/log.service.spec.tsindex e3138b895..7eda9c4a3 100644--- a/src-ui/src/app/services/rest/log.service.spec.ts+++ b/src-ui/src/app/services/rest/log.service.spec.ts@@ -49,4 +49,14 @@ describe('LogService', () => {)expect(req.request.method).toEqual('GET')})++ it('should pass limit param on logs get when provided', () => {+ const id: string = 'mail'+ const limit: number = 100+ subscription = service.get(id, limit).subscribe()+ const req = httpTestingController.expectOne(+ `${environment.apiBaseUrl}${endpoint}/${id}/?limit=${limit}`+ )+ expect(req.request.method).toEqual('GET')+ })})diff --git a/src-ui/src/app/services/rest/log.service.ts b/src-ui/src/app/services/rest/log.service.tsindex a836fa555..d07f7cd69 100644--- a/src-ui/src/app/services/rest/log.service.ts+++ b/src-ui/src/app/services/rest/log.service.ts@@ -1,4 +1,4 @@-import { HttpClient } from '@angular/common/http'+import { HttpClient, HttpParams } from '@angular/common/http'import { Injectable, inject } from '@angular/core'import { Observable } from 'rxjs'import { environment } from 'src/environments/environment'@@ -13,7 +13,13 @@ export class LogService {return this.http.get<string[]>(`${environment.apiBaseUrl}logs/`)}- get(id: string): Observable<string[]> {- return this.http.get<string[]>(`${environment.apiBaseUrl}logs/${id}/`)+ get(id: string, limit?: number): Observable<string[]> {+ let params = new HttpParams()+ if (limit !== undefined) {+ params = params.set('limit', limit.toString())+ }+ return this.http.get<string[]>(`${environment.apiBaseUrl}logs/${id}/`, {+ params,+ })}}diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.pyindex 3f7b2c385..8145e4793 100644--- a/src/documents/tests/test_api_documents.py+++ b/src/documents/tests/test_api_documents.py@@ -2250,6 +2250,23 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):self.assertEqual(response.status_code, status.HTTP_200_OK)self.assertListEqual(response.data, ["test", "test2"])+ def test_get_log_with_limit(self):+ log_data = "test1\ntest2\ntest3\n"+ with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:+ f.write(log_data)+ response = self.client.get("/api/logs/paperless/", {"limit": 2})+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertListEqual(response.data, ["test2", "test3"])++ def test_get_log_with_invalid_limit(self):+ log_data = "test1\ntest2\n"+ with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:+ f.write(log_data)+ response = self.client.get("/api/logs/paperless/", {"limit": "abc"})+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)+ response = self.client.get("/api/logs/paperless/", {"limit": -5})+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)+def test_invalid_regex_other_algorithm(self):for endpoint in ["correspondents", "tags", "document_types"]:response = self.client.post(diff --git a/src/documents/views.py b/src/documents/views.pyindex 761cba4db..ec347a553 100644--- a/src/documents/views.py+++ b/src/documents/views.py@@ -6,6 +6,7 @@ import reimport tempfileimport zipfilefrom collections import defaultdict+from collections import dequefrom datetime import datetimefrom pathlib import Pathfrom time import mktime@@ -70,6 +71,7 @@ from rest_framework import parsersfrom rest_framework import serializersfrom rest_framework.decorators import actionfrom rest_framework.exceptions import NotFound+from rest_framework.exceptions import ValidationErrorfrom rest_framework.filters import OrderingFilterfrom rest_framework.filters import SearchFilterfrom rest_framework.generics import GenericAPIView@@ -1363,6 +1365,13 @@ class UnifiedSearchViewSet(DocumentViewSet):type=OpenApiTypes.STR,location=OpenApiParameter.PATH,),+ OpenApiParameter(+ name="limit",+ type=OpenApiTypes.INT,+ location=OpenApiParameter.QUERY,+ description="Return only the last N entries from the log file",+ required=False,+ ),],responses={(200, "application/json"): serializers.ListSerializer(@@ -1394,8 +1403,22 @@ class LogViewSet(ViewSet):if not log_file.is_file():raise Http404+ limit_param = request.query_params.get("limit")+ if limit_param is not None:+ try:+ limit = int(limit_param)+ except (TypeError, ValueError):+ raise ValidationError({"limit": "Must be a positive integer"})+ if limit < 1:+ raise ValidationError({"limit": "Must be a positive integer"})+ else:+ limit = None+with log_file.open() as f:- lines = [line.rstrip() for line in f.readlines()]+ if limit is None:+ lines = [line.rstrip() for line in f.readlines()]+ else:+ lines = [line.rstrip() for line in deque(f, maxlen=limit)]return Response(lines)