A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Performance: faster statistics panel on dashboard (#11760)
Antoine Mérinodf07b8a03e1d
diff --git a/src/documents/migrations/0007_document_content_length.py b/src/documents/migrations/0007_document_content_length.pynew file mode 100644index 000000000..c294afca5--- /dev/null+++ b/src/documents/migrations/0007_document_content_length.py@@ -0,0 +1,25 @@+# Generated by Django 5.2.6 on 2026-01-24 07:33++import django.db.models.functions.text+from django.db import migrations+from django.db import models+++class Migration(migrations.Migration):+ dependencies = [+ ("documents", "0006_alter_document_checksum_unique"),+ ]++ operations = [+ migrations.AddField(+ model_name="document",+ name="content_length",+ field=models.GeneratedField(+ db_persist=True,+ expression=django.db.models.functions.text.Length("content"),+ null=False,+ help_text="Length of the content field in characters. Automatically maintained by the database for faster statistics computation.",+ output_field=models.PositiveIntegerField(default=0),+ ),+ ),+ ]diff --git a/src/documents/models.py b/src/documents/models.pyindex fe41796bd..ad5e66fe9 100644--- a/src/documents/models.py+++ b/src/documents/models.py@@ -20,7 +20,9 @@ if settings.AUDIT_LOG_ENABLED:from auditlog.registry import auditlogfrom django.db.models import Case+from django.db.models import PositiveIntegerFieldfrom django.db.models.functions import Cast+from django.db.models.functions import Lengthfrom django.db.models.functions import Substrfrom django_softdelete.models import SoftDeleteModel@@ -192,6 +194,15 @@ class Document(SoftDeleteModel, ModelWithOwner):),)+ content_length = models.GeneratedField(+ expression=Length("content"),+ output_field=PositiveIntegerField(default=0),+ db_persist=True,+ null=False,+ serialize=False,+ help_text="Length of the content field in characters. Automatically maintained by the database for faster statistics computation.",+ )+mime_type = models.CharField(_("mime type"), max_length=256, editable=False)tags = models.ManyToManyField(@@ -945,7 +956,7 @@ if settings.AUDIT_LOG_ENABLED:auditlog.register(Document,m2m_fields={"tags"},- exclude_fields=["modified"],+ exclude_fields=["content_length", "modified"],)auditlog.register(Correspondent)auditlog.register(Tag)diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.pyindex f40ef157f..96d22dc2c 100644--- a/src/documents/tests/test_api_documents.py+++ b/src/documents/tests/test_api_documents.py@@ -131,6 +131,10 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):self.assertIn("content", results_full[0])self.assertIn("id", results_full[0])+ # Content length is used internally for performance reasons.+ # No need to expose this field.+ self.assertNotIn("content_length", results_full[0])+response = self.client.get("/api/documents/?fields=id", format="json")self.assertEqual(response.status_code, status.HTTP_200_OK)results = response.data["results"]diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.pyindex 81262779a..c2a1360ca 100644--- a/src/documents/tests/test_management_exporter.py+++ b/src/documents/tests/test_management_exporter.py@@ -241,6 +241,10 @@ class TestExportImport(checksum = hashlib.md5(f.read()).hexdigest()self.assertEqual(checksum, element["fields"]["checksum"])+ # Generated field "content_length" should not be exported,+ # it is automatically computed during import.+ self.assertNotIn("content_length", element["fields"])+if document_exporter.EXPORTER_ARCHIVE_NAME in element:fname = (self.target / element[document_exporter.EXPORTER_ARCHIVE_NAME]diff --git a/src/documents/views.py b/src/documents/views.pyindex c9ac9c848..88c9c5cf7 100644--- a/src/documents/views.py+++ b/src/documents/views.py@@ -35,7 +35,6 @@ from django.db.models import Modelfrom django.db.models import Qfrom django.db.models import Sumfrom django.db.models import When-from django.db.models.functions import Lengthfrom django.db.models.functions import Lowerfrom django.db.models.manager import Managerfrom django.http import FileResponse@@ -2326,23 +2325,19 @@ class StatisticsView(GenericAPIView):user = request.user if request.user is not None else Nonedocuments = (- (- Document.objects.all()- if user is None- else get_objects_for_user_owner_aware(- user,- "documents.view_document",- Document,- )+ Document.objects.all()+ if user is None+ else get_objects_for_user_owner_aware(+ user,+ "documents.view_document",+ Document,)- .only("mime_type", "content")- .prefetch_related("tags"))tags = (Tag.objects.all()if user is Noneelse get_objects_for_user_owner_aware(user, "documents.view_tag", Tag)- )+ ).only("id", "is_inbox_tag")correspondent_count = (Correspondent.objects.count()if user is None@@ -2371,32 +2366,34 @@ class StatisticsView(GenericAPIView):).count())- documents_total = documents.count()-- inbox_tags = tags.filter(is_inbox_tag=True)+ inbox_tag_pks = list(+ tags.filter(is_inbox_tag=True).values_list("pk", flat=True),+ )documents_inbox = (- documents.filter(tags__id__in=inbox_tags).distinct().count()- if inbox_tags.exists()+ documents.filter(tags__id__in=inbox_tag_pks).values("id").distinct().count()+ if inbox_tag_pkselse None)- document_file_type_counts = (+ # Single SQL request for document stats and mime type counts+ mime_type_stats = list(documents.values("mime_type")- .annotate(mime_type_count=Count("mime_type"))- .order_by("-mime_type_count")- if documents_total > 0- else []- )-- character_count = (- documents.annotate(- characters=Length("content"),+ .annotate(+ mime_type_count=Count("id"),+ mime_type_chars=Sum("content_length"),)- .aggregate(Sum("characters"))- .get("characters__sum")+ .order_by("-mime_type_count"),)+ # Calculate totals from grouped results+ documents_total = sum(row["mime_type_count"] for row in mime_type_stats)+ character_count = sum(row["mime_type_chars"] or 0 for row in mime_type_stats)+ document_file_type_counts = [+ {"mime_type": row["mime_type"], "mime_type_count": row["mime_type_count"]}+ for row in mime_type_stats+ ]+current_asn = Document.objects.aggregate(Max("archive_serial_number", default=0),).get(@@ -2408,11 +2405,9 @@ class StatisticsView(GenericAPIView):"documents_total": documents_total,"documents_inbox": documents_inbox,"inbox_tag": (- inbox_tags.first().pk if inbox_tags.exists() else None+ inbox_tag_pks[0] if inbox_tag_pks else None), # backwards compatibility- "inbox_tags": (- [tag.pk for tag in inbox_tags] if inbox_tags.exists() else None- ),+ "inbox_tags": (inbox_tag_pks if inbox_tag_pks else None),"document_file_type_counts": document_file_type_counts,"character_count": character_count,"tag_count": len(tags),