A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Enhancement: relative date custom field queries (#14549)
shamoon67c194978e2d
diff --git a/docs/api.md b/docs/api.mdindex 91e8492ed..8a440545f 100644--- a/docs/api.md+++ b/docs/api.md@@ -162,6 +162,17 @@ including `gt` (>), `gte` (>=), `lt` (<), `lte` (<=), and `range`.Lastly, document link fields support a `contains` operator that behaveslike a "is superset of" check.+Date fields also accept relative expressions supported by+[dateparser](https://dateparser.readthedocs.io/en/stable/), such as `today`,+`in 30 days`, or `1 month ago`. For example, dates within the next 30 days:++```json+["expiry", "range", ["today", "in 30 days"]]+```++Expressions resolve each time the query runs, using `PAPERLESS_TIME_ZONE` and+`PAPERLESS_DATE_PARSER_LANGUAGES` (automatic language detection when unset).+### `/api/search/autocomplete/`Get auto completions for a partial search term.diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.htmlindex 6923df150..b04b207ad 100644--- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html+++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html@@ -33,21 +33,54 @@<ng-template #comparisonValueTemplate let-atom="atom">@if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Date) {- <input class="form-control" placeholder="yyyy-mm-dd"- [(ngModel)]="atom.value"- [ngModelOptions]="{standalone: true}"- ngbDatepicker- #d="ngbDatepicker"- [footerTemplate]="datePickerFooterTemplate" />- <button class="btn btn-sm btn-outline-secondary rounded-end" (click)="d.toggle()" type="button" aria-label="Open date picker" i18n-aria-label>- <i-bs name="calendar-event"></i-bs>- </button>- <ng-template #datePickerFooterTemplate>- <div class="btn-group-xs border-top p-2 d-flex">- <button type="button" class="btn btn-primary" (click)="atom.value = today; d.close()" i18n>Today</button>- <button type="button" class="btn btn-secondary ms-auto" (click)="d.close()" i18n>Close</button>+ @let relativeDate = getRelativeDate(atom);+ <div class="d-flex gap-2 w-100 ms-0 mt-1">+ <fieldset class="btn-group btn-group-sm flex-shrink-0 date-mode" aria-label="Date mode" i18n-aria-label>+ <input type="radio" class="btn-check" [id]="atom.id + '_fixed'" [name]="atom.id + '_dateMode'" [checked]="!relativeDate" (change)="toggleRelativeDate(atom)" [disabled]="disabled" aria-label="Fixed date" i18n-aria-label>+ <label class="btn btn-outline-secondary text-nowrap" [for]="atom.id + '_fixed'" title="Fixed date" i18n-title>+ <i-bs name="calendar"></i-bs><span class="d-none d-md-inline ms-2" i18n>Fixed</span>+ </label>+ <input type="radio" class="btn-check" [id]="atom.id + '_relative'" [name]="atom.id + '_dateMode'" [checked]="!!relativeDate" (change)="toggleRelativeDate(atom)" [disabled]="disabled" aria-label="Relative date" i18n-aria-label>+ <label class="btn btn-outline-secondary text-nowrap" [for]="atom.id + '_relative'" title="Relative date" i18n-title>+ <i-bs name="clock-history"></i-bs><span class="d-none d-md-inline ms-2" i18n>Relative</span>+ </label>+ </fieldset>+ <div class="input-group input-group-sm flex-grow-1">+ @if (relativeDate) {+ @if (relativeDate.unit.includes('{n}')) {+ <input class="form-control w-25 flex-grow-0" type="number" min="0" aria-label="Amount" i18n-aria-label+ [ngModel]="relativeDate.amount"+ (ngModelChange)="updateRelativeDate(atom, { amount: $event })"+ [ngModelOptions]="{standalone: true}" />+ }+ <label class="visually-hidden" [for]="atom.id + '_unit'" i18n>Unit</label>+ <select class="form-select" [id]="atom.id + '_unit'"+ [ngModel]="relativeDate.unit"+ (ngModelChange)="updateRelativeDate(atom, { unit: $event })"+ [ngModelOptions]="{standalone: true}">+ @for (unit of relativeDateUnits; track unit.id) {+ <option [ngValue]="unit.id">{{unit.label}}</option>+ }+ </select>+ } @else {+ <input class="form-control" placeholder="yyyy-mm-dd"+ [(ngModel)]="atom.value"+ [ngModelOptions]="{standalone: true}"+ ngbDatepicker+ #d="ngbDatepicker"+ [footerTemplate]="datePickerFooterTemplate" />+ <button class="btn btn-sm btn-outline-secondary" (click)="d.toggle()" type="button" aria-label="Open date picker" i18n-aria-label>+ <i-bs name="calendar-event"></i-bs>+ </button>+ <ng-template #datePickerFooterTemplate>+ <div class="btn-group-xs border-top p-2 d-flex">+ <button type="button" class="btn btn-primary" (click)="atom.value = today; d.close()" i18n>Today</button>+ <button type="button" class="btn btn-secondary ms-auto" (click)="d.close()" i18n>Close</button>+ </div>+ </ng-template>+ }</div>- </ng-template>+ </div>} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Float || getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Integer) {<input class="w-25 form-control rounded-end" type="number" [(ngModel)]="atom.value" [ngModelOptions]="{standalone: true}" [disabled]="disabled">} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Boolean) {@@ -83,6 +116,7 @@</ng-template><ng-template #queryAtom let-atom="atom">+ <label class="visually-hidden" [for]="atom.id + '_operator'" i18n>Operator</label><div class="input-group input-group-sm"><ng-select #fieldSelectsclass="paperless-input-select"@@ -96,7 +130,7 @@[appendTo]="selectAppendTo"(mousedown)="$event.stopImmediatePropagation()"></ng-select>- <select class="w-25 form-select" [(ngModel)]="atom.operator" [ngModelOptions]="{standalone: true}" [disabled]="disabled">+ <select class="w-25 form-select" [id]="atom.id + '_operator'" [class.rounded-end]="hasDateValueRow(atom)" [(ngModel)]="atom.operator" [ngModelOptions]="{standalone: true}" [disabled]="disabled">@for (operator of getOperatorsForField(atom.field); track operator.label) {<option [ngValue]="operator.value">{{operator.label}}</option>}@@ -151,10 +185,10 @@<input class="w-25 form-control rounded-end" type="text" [(ngModel)]="atom.value" [ngModelOptions]="{standalone: true}" [disabled]="disabled">}}- <button class="btn btn-link btn-sm text-danger pe-0" type="button" (click)="removeElement(atom)" [disabled]="disabled" aria-label="Remove query" i18n-aria-label>- <i-bs name="x-circle"></i-bs>- </button></div>+ <button class="btn btn-link btn-sm text-danger pe-0 align-self-center" type="button" (click)="removeElement(atom)" [disabled]="disabled" aria-label="Remove query" i18n-aria-label>+ <i-bs name="x-circle"></i-bs>+ </button></ng-template><ng-template #queryExpression let-expression="expression">diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scss b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scssindex f38bb4002..51d0a651a 100644--- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scss+++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scss@@ -5,6 +5,11 @@}}+:host {+ --ng-select-border-radius: var(--bs-border-radius);+ --ng-select-height: calc(1.5em + 0.5rem + calc(var(--bs-border-width) * 2));+}+:host ::ng-deep .ng-select-container {border-top-right-radius: 0 !important;border-bottom-right-radius: 0 !important;@@ -41,3 +46,13 @@min-width: 140px;}}++.date-mode .btn {+ --bs-btn-color: var(--bs-secondary-color);+ --bs-btn-hover-bg: transparent;+ --bs-btn-hover-color: var(--bs-body-color);+ --bs-btn-hover-border-color: var(--bs-border-color);+ --bs-btn-active-bg: transparent;+ --bs-btn-active-color: var(--bs-primary);+ --bs-btn-active-border-color: var(--bs-border-color);+}diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.ts b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.tsindex d1e755032..7062be77a 100644--- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.ts+++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.ts@@ -3,7 +3,7 @@ import { provideHttpClientTesting } from '@angular/common/http/testing'import { LOCALE_ID } from '@angular/core'import { ComponentFixture, TestBed } from '@angular/core/testing'import { FormsModule, ReactiveFormsModule } from '@angular/forms'-import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'+import { NgbDateAdapter, NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'import { NgSelectModule } from '@ng-select/ng-select'import { allIcons, NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'import { of } from 'rxjs'@@ -11,6 +11,7 @@ import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field'import {CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP,CustomFieldQueryLogicalOperator,+ CustomFieldQueryOperator,CustomFieldQueryOperatorGroups,} from 'src/app/data/custom-field-query'import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'@@ -19,6 +20,7 @@ import {CustomFieldQueryElement,CustomFieldQueryExpression,} from 'src/app/utils/custom-field-query-element'+import { ISODateAdapter } from 'src/app/utils/ngb-iso-date-adapter'import {CustomFieldQueriesModel,CustomFieldsQueryDropdownComponent,@@ -48,6 +50,12 @@ const customFields = [data_type: CustomFieldDataType.Monetary,extra_data: { default_currency: 'EUR' },},+ {+ id: 4,+ name: 'Test Date Field',+ data_type: CustomFieldDataType.Date,+ extra_data: {},+ },]describe('CustomFieldsQueryDropdownComponent', () => {@@ -69,6 +77,7 @@ describe('CustomFieldsQueryDropdownComponent', () => {provideHttpClient(withInterceptorsFromDi()),provideHttpClientTesting(),{ provide: LOCALE_ID, useValue: 'de' },+ { provide: NgbDateAdapter, useClass: ISODateAdapter },],}).compileComponents()@@ -237,6 +246,74 @@ describe('CustomFieldsQueryDropdownComponent', () => {expect(component.selectionModel.queries.length).toBe(1)})+ it('should build relative date values from amount and unit', async () => {+ component.useDropdown = false+ const atom = new CustomFieldQueryAtom([+ 4,+ CustomFieldQueryOperator.GreaterThanOrEqual,+ '2026-10-22',+ ])+ component.selectionModel.addAtom(atom)+ fixture.detectChanges()+ await fixture.whenStable()+ const query = (selector: string) =>+ fixture.nativeElement.querySelector(selector)+ expect(query('[ngbDatepicker]')).not.toBeNull()+ expect(query('input[id$="_fixed"]').checked).toBe(true)++ query('input[id$="_relative"]').click()+ fixture.detectChanges()+ await fixture.whenStable()+ expect(atom.value).toBeNull()+ const amount: HTMLInputElement = query('input[aria-label="Amount"]')+ const unit: HTMLSelectElement = query('select[id$="_unit"]')+ amount.value = '30'+ amount.dispatchEvent(new Event('input'))+ expect(atom.serialize()).toEqual([4, 'gte', '30 days ago'])+ unit.selectedIndex = 6+ unit.dispatchEvent(new Event('change'))+ expect(atom.serialize()).toEqual([4, 'gte', 'in 30 weeks'])+ unit.selectedIndex = 0+ unit.dispatchEvent(new Event('change'))+ fixture.detectChanges()+ expect(atom.serialize()).toEqual([4, 'gte', 'today'])+ expect(query('input[aria-label="Amount"]')).toBeNull()++ query('input[id$="_fixed"]').click()+ fixture.detectChanges()+ expect(atom.value).toBeNull()+ expect(query('[ngbDatepicker]')).not.toBeNull()+ })++ it('should restore a saved relative date value', async () => {+ component.useDropdown = false+ const atom = new CustomFieldQueryAtom([+ 4,+ CustomFieldQueryOperator.Exact,+ 'in 2 months',+ ])+ component.selectionModel.addAtom(atom)+ fixture.detectChanges()+ await fixture.whenStable()+ expect(component.getRelativeDate(atom)).toEqual({+ amount: 2,+ unit: 'in {n} months',+ })+ expect(+ fixture.nativeElement.querySelector('input[aria-label="Amount"]').value+ ).toBe('2')++ const today = new CustomFieldQueryAtom([+ 4,+ CustomFieldQueryOperator.Exact,+ 'today',+ ])+ expect(component.getRelativeDate(today)).toEqual({+ amount: null,+ unit: 'today',+ })+ })+describe('CustomFieldQueriesModel', () => {let model: CustomFieldQueriesModeldiff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.tsindex 121e396e1..874655e11 100644--- a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts+++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts@@ -49,6 +49,26 @@ import { LoadingComponentWithPermissions } from '../../loading-component/loadingimport { ClearableBadgeComponent } from '../clearable-badge/clearable-badge.component'import { DocumentLinkComponent } from '../input/document-link/document-link.component'+interface RelativeDate {+ // The expression sent to the API, with {n} standing in for the amount+ amount: number | null+ unit: string+}++function parseRelativeDate(+ value: string,+ unit: string+): RelativeDate | undefined {+ const [prefix, suffix] = unit.split('{n}')+ if (suffix === undefined) {+ return value === unit ? { amount: null, unit } : undefined+ }+ if (!value.startsWith(prefix) || !value.endsWith(suffix)) return undefined+ const amount = value.slice(prefix.length, value.length - suffix.length)+ if (!amount || !Number.isInteger(+amount) || +amount < 0) return undefined+ return { amount: +amount, unit }+}+export class CustomFieldQueriesModel {private readonly _queries = signal<CustomFieldQueryElement[]>([])private rootSubscriptions: Subscription[] = []@@ -305,6 +325,67 @@ export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPermpublic readonly today: string = new Date().toLocaleDateString('en-CA')+ public readonly relativeDateUnits = [+ { id: 'today', label: $localize`Today` },+ { id: '{n} days ago', label: $localize`days ago` },+ { id: '{n} weeks ago', label: $localize`weeks ago` },+ { id: '{n} months ago', label: $localize`months ago` },+ { id: '{n} years ago', label: $localize`years ago` },+ { id: 'in {n} days', label: $localize`days from now` },+ { id: 'in {n} weeks', label: $localize`weeks from now` },+ { id: 'in {n} months', label: $localize`months from now` },+ { id: 'in {n} years', label: $localize`years from now` },+ ]++ private readonly relativeDates = new Map<string, RelativeDate>()++ getRelativeDate(atom: CustomFieldQueryAtom): RelativeDate | undefined {+ if (!this.relativeDates.has(atom.id) && typeof atom.value === 'string') {+ for (const { id } of this.relativeDateUnits) {+ const relativeDate = parseRelativeDate(atom.value, id)+ if (relativeDate) {+ this.relativeDates.set(atom.id, relativeDate)+ break+ }+ }+ }+ return this.relativeDates.get(atom.id)+ }++ hasDateValueRow(atom: CustomFieldQueryAtom): boolean {+ return (+ this.getCustomFieldByID(atom.field)?.data_type ===+ CustomFieldDataType.Date &&+ atom.operator !== CustomFieldQueryOperator.Exists &&+ atom.operator !== CustomFieldQueryOperator.IsNull+ )+ }++ toggleRelativeDate(atom: CustomFieldQueryAtom) {+ if (this.getRelativeDate(atom)) {+ this.relativeDates.delete(atom.id)+ atom.value = null+ } else {+ this.relativeDates.set(atom.id, { amount: null, unit: '{n} days ago' })+ this.updateRelativeDate(atom, {})+ }+ }++ updateRelativeDate(+ atom: CustomFieldQueryAtom,+ change: Partial<RelativeDate>+ ) {+ const { amount, unit } = Object.assign(+ this.relativeDates.get(atom.id),+ change+ )+ if (!unit.includes('{n}')) {+ atom.value = unit+ } else {+ atom.value = amount === null ? null : unit.replace('{n}', `${amount}`)+ }+ }+public customFieldSearchFn = (term: string, field: CustomField): boolean =>matchesSearchText(field?.name, term)diff --git a/src/documents/filters.py b/src/documents/filters.pyindex a95e3b288..00a47552d 100644--- a/src/documents/filters.py+++ b/src/documents/filters.py@@ -11,6 +11,8 @@ from decimal import InvalidOperationfrom typing import TYPE_CHECKINGfrom typing import Any+import dateparser+from django.conf import settingsfrom django.contrib.contenttypes.models import ContentTypefrom django.db.models import Casefrom django.db.models import CharField@@ -25,6 +27,7 @@ from django.db.models import Valuefrom django.db.models import Whenfrom django.db.models.functions import Castfrom django.db.models.functions import NullIf+from django.utils import timezonefrom django.utils.translation import gettext_lazy as _from django_filters import DateFilterfrom django_filters.rest_framework import BooleanFilter@@ -57,6 +60,7 @@ from documents.versioning import annotate_effective_contentif TYPE_CHECKING:from collections.abc import Callable+ from datetime import datetimeCHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]ID_KWARGS = ["in", "exact"]@@ -322,6 +326,51 @@ class MonetaryAmountField(serializers.Field):return str(value)+class DateQueryField(serializers.DateField):+ """+ Accepts either a fixed date or a relative date expression using `dateparser` and+ resolves it against the provided reference datetime.+ """++ default_error_messages = {+ "expression": _(+ 'Expected a date or a relative date such as "today" or "in 30 days".',+ ),+ }++ def __init__(self, *, reference_datetime: datetime, **kwargs):+ super().__init__(**kwargs)+ self.reference_datetime = reference_datetime++ def to_internal_value(self, value):+ try:+ # Let the parent class handle fixed-date formats and string validation first+ return super().to_internal_value(value)+ except serializers.ValidationError:+ if not isinstance(value, str):+ raise++ try:+ parsed = dateparser.parse(+ value,+ # The UI always builds English expressions+ locales=[*settings.DATE_PARSER_LANGUAGES, "en"]+ if settings.DATE_PARSER_LANGUAGES+ else None,+ settings={+ "PARSERS": ["relative-time"],+ "RELATIVE_BASE": self.reference_datetime,+ "TIMEZONE": str(self.reference_datetime.tzinfo),+ "RETURN_AS_TIMEZONE_AWARE": True,+ },+ )+ except (ValueError, OverflowError):+ self.fail("expression")+ if parsed is None:+ self.fail("expression")+ return parsed.astimezone(self.reference_datetime.tzinfo).date()++class SelectField(serializers.CharField):def __init__(self, custom_field: CustomField) -> None:self._options = custom_field.extra_data["select_options"]@@ -463,6 +512,8 @@ class CustomFieldQueryParser:raise serializers.ValidationError({self._validation_prefix: [_("Value must be valid JSON.")]},)+ # Save the current local time as the reference for the query+ self._reference_datetime = timezone.localtime()return (self._parse_expr(expr, validation_prefix=self._validation_prefix),self._annotations,@@ -649,6 +700,8 @@ class CustomFieldQueryParser:):# DateField admits queries in the form of `year__exact`, etc. These take integers.field = serializers.IntegerField()+ elif custom_field.data_type == CustomField.FieldDataType.DATE:+ field = DateQueryField(reference_datetime=self._reference_datetime)elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:# We can be more specific here and make sure the value is a list.field = serializers.ListField(child=serializers.IntegerField())diff --git a/src/documents/tests/test_api_filter_by_custom_fields.py b/src/documents/tests/test_api_filter_by_custom_fields.pyindex 3c0ace782..96123fb17 100644--- a/src/documents/tests/test_api_filter_by_custom_fields.py+++ b/src/documents/tests/test_api_filter_by_custom_fields.py@@ -2,8 +2,12 @@ import jsonimport typesfrom collections.abc import Callablefrom datetime import date+from unittest.mock import patchfrom urllib.parse import quote+import time_machine+from django.test import override_settings+from django.utils import timezonefrom rest_framework.test import APITestCasefrom documents.models import CustomField@@ -443,6 +447,65 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):),)+ @override_settings(TIME_ZONE="America/Los_Angeles")+ def test_relative_dates(self) -> None:+ # Reevaluate the same mixed relative/fixed range on different local dates.+ for today in (date(2024, 8, 22), date(2024, 11, 15)):+ with (+ self.subTest(today=today),+ time_machine.travel(f"{today}T23:30:00-07:00", tick=False),+ patch(+ "documents.filters.timezone.localtime",+ wraps=timezone.localtime,+ ) as localtime,+ ):+ self._assert_query_match_predicate(+ ["date_field", "range", ["today", "2024-11-15"]],+ lambda document, today=today: (+ "date_field" in document+ and document["date_field"] is not None+ and today <= document["date_field"] <= date(2024, 11, 15)+ ),+ )+ localtime.assert_called_once_with()++ def test_relative_date_validation(self) -> None:+ for value, keyword in (+ ("not a date", "date"),+ ("January 2024", "date"),+ ("today + " + "9" * 5000 + " days", "date"),+ (True, "format"),+ ):+ with self.subTest(value=str(value)[:50]):+ self._assert_validation_error(+ json.dumps(["date_field", "exact", value]),+ ["custom_field_query", "2"],+ keyword,+ )++ @time_machine.travel("2024-08-22T12:00:00Z", tick=False)+ def test_relative_date_languages(self) -> None:+ for locales in (["de"], ["zh-Hans", "de"], None):+ with (+ self.subTest(locales=locales),+ override_settings(+ DATE_PARSER_LANGUAGES=locales,+ ),+ ):+ self._assert_query_match_predicate(+ ["date_field", "exact", "heute"],+ lambda document: (+ "date_field" in document+ and document["date_field"] == date(2024, 8, 22)+ ),+ )+ with override_settings(DATE_PARSER_LANGUAGES=["en"]):+ self._assert_validation_error(+ json.dumps(["date_field", "exact", "heute"]),+ ["custom_field_query", "2"],+ "date",+ )+def test_gt_monetary(self) -> None:self._assert_query_match_predicate(["monetary_field", "gt", "99"],