A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Fix: exclude next-period start from relative date-range filters (#13381)
Trenton H1b32b9d67869
diff --git a/src/documents/search/_dates.py b/src/documents/search/_dates.pyindex a94935022..dbdf6b42b 100644--- a/src/documents/search/_dates.py+++ b/src/documents/search/_dates.py@@ -43,8 +43,16 @@ def _fmt(dt: datetime) -> str:def _iso_range(lo: datetime, hi: datetime) -> str:- """Format a [lo TO hi] range string in ISO 8601 for Tantivy query syntax."""- return f"[{_fmt(lo)} TO {_fmt(hi)}]"+ """+ Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax.++ ``hi`` is always the exclusive ceiling of a computed period (the start of+ the *next* day/week/month/quarter/year), so the closing bracket must be+ the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the+ first instant of the following period (e.g. the 1st of next month) is+ incorrectly included in the match.+ """+ return f"[{_fmt(lo)} TO {_fmt(hi)}}}"def _quarter_start(d: date) -> date:diff --git a/src/documents/search/_translate.py b/src/documents/search/_translate.pyindex 7afa23773..a3ec729fd 100644--- a/src/documents/search/_translate.py+++ b/src/documents/search/_translate.py@@ -566,6 +566,17 @@ def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:lo_pair, hi_pair = hi_pair, lo_pairlo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO- hi_iso = _fmt(hi_pair[1]) if hi_pair is not None else OPEN_HI- return f"{field}:[{lo_iso} TO {hi_iso}]"+ # A bound resolves to (floor, ceil) where floor == ceil for an exact instant+ # (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for+ # a coarser period token (year/month/day precision). Only the latter needs a+ # half-open close: its ceil is the start of the *next* period and must be+ # excluded, or that instant (e.g. the 1st of next month) wrongly matches.+ if hi_pair is not None:+ hi_iso = _fmt(hi_pair[1])+ hi_close = "]" if hi_pair[0] == hi_pair[1] else "}"+ else:+ hi_iso = OPEN_HI+ hi_close = "]"++ return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}"diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.pyindex 60ba2bf2f..2b7112047 100644--- a/src/documents/tests/search/test_query.py+++ b/src/documents/tests/search/test_query.py@@ -32,7 +32,9 @@ AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summerdef _range(result: str, field: str) -> tuple[str, str]:- m = re.search(rf"{field}:\[(.+?) TO (.+?)\]", result)+ # Half-open period ranges close with "}" (exclusive); exact-instant ranges+ # (full ISO datetimes, "now", relative offsets) close with "]" (inclusive).+ m = re.search(rf"{field}:\[(.+?) TO (.+?)[\]}}]", result)assert m, f"No range for {field!r} in: {result!r}"return m.group(1), m.group(2)diff --git a/src/documents/tests/search/test_translate.py b/src/documents/tests/search/test_translate.pyindex ff91cf301..70611aecc 100644--- a/src/documents/tests/search/test_translate.py+++ b/src/documents/tests/search/test_translate.py@@ -214,27 +214,27 @@ class TestTranslateScalar:("created","2020",- "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",+ "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",),("created","202003",- "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z]",+ "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",),("created","20200115",- "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z]",+ "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",),("created","2020-01-15",- "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z]",+ "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",),("created","2020-03",- "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z]",+ "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",),],)@@ -248,9 +248,9 @@ class TestTranslateScalar:assert exc_info.value.value == "202023"def test_keyword_delegates(self) -> None:- # keyword path produces a range; just assert it is a created range+ # keyword path produces a half-open range; just assert it is a created rangeout = translate_scalar("created", "today", UTC)- assert out.startswith("created:[") and out.endswith("]")+ assert out.startswith("created:[") and out.endswith("}")def test_14digit_compact_datetime(self) -> None:out = translate_scalar("created", "20240115120000", UTC)@@ -279,21 +279,21 @@ class TestTranslateRange:@pytest.mark.parametrize(("lo", "hi", "expected"),[- ("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"),+ ("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),("202001","202006",- "created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z]",+ "created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",),("20200101","20201231",- "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",+ "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",),("2020-01-01","2020-12-31",- "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",+ "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",),],)@@ -302,7 +302,7 @@ class TestTranslateRange:def test_reversed_swaps(self):assert translate_range("created", "2009", "2005", UTC) == (- "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"+ "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}")def test_open_upper(self):@@ -311,7 +311,7 @@ class TestTranslateRange:def test_open_lower(self):out = translate_range("created", "", "2020", UTC)- assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z]"+ assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"def test_invalid_bound_raises(self):with pytest.raises(InvalidDateQuery) as exc_info:@@ -334,16 +334,16 @@ class TestTranslateQuery:[("created:2020",- "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",+ "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",),("tag:foo,bar", "tag:foo AND tag:bar"),# 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)("tag:foo,type:bar", "tag:foo AND document_type:bar"),("created:[2020 TO 2021],added:[2022 TO 2023]",- "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z]"+ "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"" AND "- "added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z]",+ "added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}",),# correspondent is not multi-value: comma stays literal inside the value("correspondent:foo,bar", "correspondent:foo,bar"),@@ -506,7 +506,7 @@ class TestOperatorNormalization:def test_date_range_preserved(self) -> None:out = translate_query("created:[2020 TO 2021]", UTC)# Must not corrupt the ISO range- assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z]"+ assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"def test_date_scalar_with_or(self) -> None:out = translate_query("created:2020 OR foo", UTC)@@ -581,42 +581,42 @@ class TestKeywordDateResolution:[pytest.param("today",- "created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z]",+ "created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",id="today",),pytest.param("yesterday",- "created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z]",+ "created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",id="yesterday",),pytest.param("previous week",- "created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z]",+ "created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",id="previous-week",),pytest.param("this month",- "created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z]",+ "created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",id="this-month",),pytest.param("previous month",- "created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z]",+ "created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",id="previous-month",),pytest.param("this year",- "created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z]",+ "created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",id="this-year",),pytest.param("previous year",- "created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z]",+ "created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",id="previous-year",),pytest.param("previous quarter",- "created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z]",+ "created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",id="previous-quarter",),],@@ -637,42 +637,42 @@ class TestKeywordDateResolution:[pytest.param("today",- "added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z]",+ "added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",id="today",),pytest.param("yesterday",- "added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z]",+ "added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",id="yesterday",),pytest.param("previous week",- "added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z]",+ "added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",id="previous-week",),pytest.param("this month",- "added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z]",+ "added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",id="this-month",),pytest.param("previous month",- "added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z]",+ "added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",id="previous-month",),pytest.param("this year",- "added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z]",+ "added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",id="this-year",),pytest.param("previous year",- "added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z]",+ "added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",id="previous-year",),pytest.param("previous quarter",- "added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z]",+ "added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",id="previous-quarter",),],@@ -719,7 +719,7 @@ class TestISODatetimeBounds:def test_translate_query_text_before_comma_separated_date_clause(self) -> None:result = translate_query("schäfersee,created:previous year", UTC)assert result == (- "schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z]"+ "schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}")def test_invalid_iso_datetime_raises(self) -> None:diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.pyindex b42907132..dac1d6fb0 100644--- a/src/documents/tests/test_api_search.py+++ b/src/documents/tests/test_api_search.py@@ -720,6 +720,48 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):self.assertEqual(results[0]["id"], 3)self.assertEqual(results[0]["title"], "bank statement 3")+ def test_search_added_previous_month_excludes_next_period_start(self) -> None:+ """+ GIVEN:+ - One document added at the last instant of last month+ - One document added exactly at the first instant of this month+ WHEN:+ - Query for documents added in the previous month+ THEN:+ - Only the document from last month is returned; the document dated+ exactly at the start of this month (the exclusive upper bound of+ the range) is not+ """+ d1 = DocumentFactory.create(+ title="end of last month",+ content="last instant of last month",+ checksum="A",+ pk=1,+ added=timezone.make_aware(datetime.datetime(2024, 1, 31, 23, 59, 59)),+ )+ d2 = DocumentFactory.create(+ title="start of this month",+ content="first instant of this month",+ checksum="B",+ pk=2,+ added=timezone.make_aware(datetime.datetime(2024, 2, 1, 0, 0, 0)),+ )++ backend = get_backend()+ backend.add_or_update(d1)+ backend.add_or_update(d2)++ with time_machine.travel(+ timezone.make_aware(datetime.datetime(2024, 2, 15, 12, 0, 0)),+ tick=False,+ ):+ response = self.client.get("/api/documents/?query=added:previous month")+ results = response.data["results"]++ self.assertEqual(len(results), 1)+ self.assertEqual(results[0]["id"], 1)+ self.assertEqual(results[0]["title"], "end of last month")+def test_search_added_invalid_date(self) -> None:"""GIVEN: