A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Fix/GHSA-x395-6h48-wr8v
shamoonafaf39e43aeb
diff --git a/docs/advanced_usage.md b/docs/advanced_usage.mdindex de1068864..0b5a7b601 100644--- a/docs/advanced_usage.md+++ b/docs/advanced_usage.md@@ -431,8 +431,10 @@ This allows for complex logic to be included in the format, including [logical sand [filters](https://jinja.palletsprojects.com/en/3.1.x/templates/#id11) to manipulate the [variables](#filename-format-variables)provided. The template is provided as a string, potentially multiline, and rendered into a single line.-In addition, the entire Document instance is available to be utilized in a more advanced way, as well as some variables which only make sense to be accessed-with more complex logic.+In addition, a limited `document` object is available for advanced templates.+This object includes common metadata fields such as `id`, `pk`, `title`, `content`, `page_count`, `created`, `added`, `modified`, `mime_type`,+`checksum`, `archive_checksum`, `archive_serial_number`, `filename`, `archive_filename`, and `original_filename`.+Related values are available as nested objects with limited fields, for example document.correspondent.name, etc.#### Custom Jinja2 Filtersdiff --git a/src/documents/serialisers.py b/src/documents/serialisers.pyindex a7d852fb8..bec1254c8 100644--- a/src/documents/serialisers.py+++ b/src/documents/serialisers.py@@ -6,6 +6,7 @@ import refrom datetime import datetimefrom decimal import Decimalfrom typing import TYPE_CHECKING+from typing import Anyfrom typing import Literalimport magic@@ -73,6 +74,7 @@ from documents.models import WorkflowTriggerfrom documents.parsers import is_mime_type_supportedfrom documents.permissions import get_document_count_filter_for_userfrom documents.permissions import get_groups_with_only_permission+from documents.permissions import get_objects_for_user_owner_awarefrom documents.permissions import set_permissions_for_objectfrom documents.regex import validate_regex_patternfrom documents.templating.filepath import validate_filepath_template_and_render@@ -2753,8 +2755,22 @@ class StoragePathTestSerializer(SerializerWithPerms):)document = serializers.PrimaryKeyRelatedField(- queryset=Document.objects.all(),+ queryset=Document.objects.none(),required=True,label="Document",write_only=True,)++ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ request = self.context.get("request")+ user = getattr(request, "user", None) if request else None+ if user is not None and user.is_authenticated:+ document_field = self.fields.get("document")+ if not isinstance(document_field, serializers.PrimaryKeyRelatedField):+ return+ document_field.queryset = get_objects_for_user_owner_aware(+ user,+ "documents.view_document",+ Document,+ )diff --git a/src/documents/templating/filepath.py b/src/documents/templating/filepath.pyindex 805cefbdb..b4dd367fb 100644--- a/src/documents/templating/filepath.py+++ b/src/documents/templating/filepath.py@@ -193,6 +193,52 @@ def get_basic_metadata_context(}+def get_safe_document_context(+ document: Document,+ tags: Iterable[Tag],+) -> dict[str, object]:+ """+ Build a document context object to avoid supplying entire model instance.+ """+ return {+ "id": document.pk,+ "pk": document.pk,+ "title": document.title,+ "content": document.content,+ "page_count": document.page_count,+ "created": document.created,+ "added": document.added,+ "modified": document.modified,+ "archive_serial_number": document.archive_serial_number,+ "mime_type": document.mime_type,+ "checksum": document.checksum,+ "archive_checksum": document.archive_checksum,+ "filename": document.filename,+ "archive_filename": document.archive_filename,+ "original_filename": document.original_filename,+ "owner": {"username": document.owner.username, "id": document.owner.id}+ if document.owner+ else None,+ "tags": [{"name": tag.name, "id": tag.id} for tag in tags],+ "correspondent": (+ {"name": document.correspondent.name, "id": document.correspondent.id}+ if document.correspondent+ else None+ ),+ "document_type": (+ {"name": document.document_type.name, "id": document.document_type.id}+ if document.document_type+ else None+ ),+ "storage_path": {+ "path": document.storage_path.path,+ "id": document.storage_path.id,+ }+ if document.storage_path+ else None,+ }++def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:"""Given an Iterable of tags, constructs some context from them for usage@@ -303,7 +349,7 @@ def validate_filepath_template_and_render(# Build the context dictionarycontext = (- {"document": document}+ {"document": get_safe_document_context(document, tags=tags_list)}| get_basic_metadata_context(document, no_value_default=NO_VALUE_PLACEHOLDER)| get_creation_date_context(document)| get_added_date_context(document)diff --git a/src/documents/tests/test_api_objects.py b/src/documents/tests/test_api_objects.pyindex 0eb99f023..12d4918c5 100644--- a/src/documents/tests/test_api_objects.py+++ b/src/documents/tests/test_api_objects.py@@ -5,10 +5,13 @@ from unittest import mockfrom django.contrib.auth.models import Permissionfrom django.contrib.auth.models import Userfrom django.test import override_settings+from guardian.shortcuts import assign_permfrom rest_framework import statusfrom rest_framework.test import APITestCasefrom documents.models import Correspondent+from documents.models import CustomField+from documents.models import CustomFieldInstancefrom documents.models import Documentfrom documents.models import DocumentTypefrom documents.models import StoragePath@@ -398,6 +401,292 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):self.assertEqual(response.status_code, status.HTTP_200_OK)self.assertEqual(response.data, "folder/Something")+ def test_test_storage_path_requires_document_view_permission(self) -> None:+ owner = User.objects.create_user(username="owner")+ unprivileged = User.objects.create_user(username="unprivileged")+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ title="Sensitive",+ checksum="123",+ )+ self.client.force_authenticate(user=unprivileged)+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": "path/{{ title }}",+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)+ self.assertIn("document", response.data)++ def test_test_storage_path_allows_shared_document_view_permission(self) -> None:+ owner = User.objects.create_user(username="owner")+ viewer = User.objects.create_user(username="viewer")+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ title="Shared",+ checksum="123",+ )+ assign_perm("view_document", viewer, document)++ self.client.force_authenticate(user=viewer)+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": "path/{{ title }}",+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "path/Shared")++ def test_test_storage_path_exposes_basic_document_context_but_not_sensitive_owner_data(+ self,+ ) -> None:+ owner = User.objects.create_user(+ username="owner",+ password="password",+ email="[email protected]",+ )+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ title="Document",+ content="Top secret content",+ page_count=2,+ checksum="123",+ )+ self.client.force_authenticate(user=owner)++ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": "{{ document.owner.username }}",+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "owner")++ for expression, expected in (+ ("{{ document.content }}", "Top secret content"),+ ("{{ document.id }}", str(document.id)),+ ("{{ document.page_count }}", "2"),+ ):+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": expression,+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, expected)++ for expression in (+ "{{ document.owner.password }}",+ "{{ document.owner.email }}",+ ):+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": expression,+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertIsNone(response.data)++ def test_test_storage_path_includes_related_objects_for_visible_document(+ self,+ ) -> None:+ owner = User.objects.create_user(username="owner")+ viewer = User.objects.create_user(username="viewer")+ private_correspondent = Correspondent.objects.create(+ name="Private Correspondent",+ owner=owner,+ )+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ correspondent=private_correspondent,+ title="Document",+ checksum="123",+ )+ assign_perm("view_document", viewer, document)++ self.client.force_authenticate(user=viewer)+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": "{{ correspondent }}",+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "Private Correspondent")++ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": (+ "{{ document.correspondent.name if document.correspondent else 'none' }}"+ ),+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "Private Correspondent")++ def test_test_storage_path_superuser_can_view_private_related_objects(self) -> None:+ owner = User.objects.create_user(username="owner")+ private_correspondent = Correspondent.objects.create(+ name="Private Correspondent",+ owner=owner,+ )+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ correspondent=private_correspondent,+ title="Document",+ checksum="123",+ )++ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": (+ "{{ document.correspondent.name if document.correspondent else 'none' }}"+ ),+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "Private Correspondent")++ def test_test_storage_path_includes_doc_type_storage_path_and_tags(+ self,+ ) -> None:+ owner = User.objects.create_user(username="owner")+ viewer = User.objects.create_user(username="viewer")+ private_document_type = DocumentType.objects.create(+ name="Private Type",+ owner=owner,+ )+ private_storage_path = StoragePath.objects.create(+ name="Private Storage Path",+ path="private/path",+ owner=owner,+ )+ private_tag = Tag.objects.create(+ name="Private Tag",+ owner=owner,+ )+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ document_type=private_document_type,+ storage_path=private_storage_path,+ title="Document",+ checksum="123",+ )+ document.tags.add(private_tag)+ assign_perm("view_document", viewer, document)++ self.client.force_authenticate(user=viewer)+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": (+ "{{ document.document_type.name if document.document_type else 'none' }}/"+ "{{ document.storage_path.path if document.storage_path else 'none' }}/"+ "{{ document.tags[0].name if document.tags else 'none' }}"+ ),+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "Private Type/private/path/Private Tag")++ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": "{{ document_type }}/{{ tag_list if tag_list else 'none' }}",+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "Private Type/Private Tag")++ def test_test_storage_path_includes_custom_fields_for_visible_document(+ self,+ ) -> None:+ owner = User.objects.create_user(username="owner")+ viewer = User.objects.create_user(username="viewer")+ document = Document.objects.create(+ mime_type="application/pdf",+ owner=owner,+ title="Document",+ checksum="123",+ )+ custom_field = CustomField.objects.create(+ name="Secret Number",+ data_type=CustomField.FieldDataType.INT,+ )+ CustomFieldInstance.objects.create(+ document=document,+ field=custom_field,+ value_int=42,+ )+ assign_perm("view_document", viewer, document)++ self.client.force_authenticate(user=viewer)+ response = self.client.post(+ f"{self.ENDPOINT}test/",+ json.dumps(+ {+ "document": document.id,+ "path": "{{ custom_fields | get_cf_value('Secret Number', 'none') }}",+ },+ ),+ content_type="application/json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.data, "42")+class TestBulkEditObjects(APITestCase):# See test_api_permissions.py for bulk tests on permissionsdiff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.pyindex befc7050f..186483655 100644--- a/src/documents/tests/test_file_handling.py+++ b/src/documents/tests/test_file_handling.py@@ -1382,11 +1382,11 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):def test_template_with_security(self):"""GIVEN:- - Filename format with one or more undefined variables+ - Filename format with an unavailable document attributeWHEN:- Filepath for a document with this format is calledTHEN:- - The first undefined variable is logged+ - The missing attribute is logged- The default format is used"""doc_a = Document.objects.create(@@ -1408,7 +1408,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):self.assertEqual(len(capture.output), 1)self.assertEqual(capture.output[0],- "WARNING:paperless.templating:Template attempted restricted operation: <bound method Model.save of <Document: 2020-06-25 Does Matter>> is not safely callable",+ "ERROR:paperless.templating:Template variable error: 'dict object' has no attribute 'save'",)def test_template_with_custom_fields(self):diff --git a/src/documents/views.py b/src/documents/views.pyindex babc4e9aa..2ce12c330 100644--- a/src/documents/views.py+++ b/src/documents/views.py@@ -2411,7 +2411,10 @@ class StoragePathViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet):"""Test storage path against a document"""- serializer = StoragePathTestSerializer(data=request.data)+ serializer = StoragePathTestSerializer(+ data=request.data,+ context={"request": request},+ )serializer.is_valid(raise_exception=True)document = serializer.validated_data.get("document")