Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

fix(nlp): harden date parsing and NER helpers

Christian0543c07a9ea3
diff --git a/src/utils/dates.py b/src/utils/dates.py
index cfe6cdb..1c00d23 100644
--- a/src/utils/dates.py
+++ b/src/utils/dates.py
@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from typing import Optional
import re
@@ -13,6 +13,16 @@ _DATE_INPUT_FORMATS = (
"%b %d %Y",
"%d %B %Y",
"%d %b %Y",
+ "%Y-%m-%dT%H:%M:%S",
+ "%Y-%m-%dT%H:%M:%S%z",
+ "%Y-%m-%dT%H:%M:%S.%f",
+ "%Y-%m-%dT%H:%M:%S.%f%z",
+ "%Y-%m-%d %H:%M:%S",
+ "%Y-%m-%d %H:%M:%S%z",
+ "%Y-%m-%d %H:%M:%S.%f",
+ "%Y-%m-%d %H:%M:%S.%f%z",
+ "%I:%M %p on %d %B, %Y",
+ "%I:%M %p on %d %b, %Y",
)
_RELATIVE_PATTERNS = (
@@ -56,15 +66,22 @@ def _shift_months(ref: datetime, delta: int) -> datetime:
return ref.replace(year=year, month=month, day=day)
+def to_naive_utc(value: datetime) -> datetime:
+ if value.tzinfo is None:
+ return value
+ return value.astimezone(timezone.utc).replace(tzinfo=None)
+
+
def parse_date_string(value: Optional[str]) -> Optional[datetime]:
if not value or not isinstance(value, str):
return None
cleaned = value.strip()
for fmt in _DATE_INPUT_FORMATS:
try:
- return datetime.strptime(cleaned, fmt)
+ parsed = datetime.strptime(cleaned, fmt)
except ValueError:
continue
+ return to_naive_utc(parsed)
return None
diff --git a/src/utils/nlp/ner.py b/src/utils/nlp/ner.py
index 50904f4..f862f59 100644
--- a/src/utils/nlp/ner.py
+++ b/src/utils/nlp/ner.py
@@ -10,6 +10,7 @@ Modified By: Christian Nonis <[email protected]>
"""
from pydantic import BaseModel
+from src.lib.tracing.profiler import profile_stage
from src.utils.nlp.spacy import MODEL_NAMES, _spacy_manager
from src.utils.nlp.lang_detect import langid_detect
@@ -101,21 +102,25 @@ class MultiLangEntityExtractor:
ignore_pos: list[str] | None = None,
) -> ExtractElementsResponse:
if lang is None:
- lang, _ = langid_detect(text)
+ with profile_stage("nlp.detect_language"):
+ lang, _ = langid_detect(text)
if lang not in MODEL_NAMES:
return {"tokens": [], "noun_chunks": []}
try:
- nlp = self.spacy_manager.get_model(lang)
+ with profile_stage("nlp.load_model", lang=lang):
+ nlp = self.spacy_manager.get_model(lang)
except Exception:
return {"tokens": [], "noun_chunks": []}
- doc = nlp(text)
+ with profile_stage("nlp.parse", chars=len(text or "")):
+ doc = nlp(text)
ignore_pos_set = (
set(ignore_pos) if ignore_pos is not None else _DEFAULT_IGNORE_POS
)
- return ExtractElementsResponse(
- tokens=self._process_doc(doc, ignore_pos_set)["tokens"],
- noun_chunks=self._process_doc(doc, ignore_pos_set)["noun_chunks"],
- )
+ with profile_stage("nlp.process_doc"):
+ tokens = self._process_doc(doc, ignore_pos_set)["tokens"]
+ with profile_stage("nlp.process_doc"):
+ noun_chunks = self._process_doc(doc, ignore_pos_set)["noun_chunks"]
+ return ExtractElementsResponse(tokens=tokens, noun_chunks=noun_chunks)
def extract_elements_batch(
self,
diff --git a/tests/test_dates.py b/tests/test_dates.py
new file mode 100644
index 0000000..aaf622a
--- /dev/null
+++ b/tests/test_dates.py
@@ -0,0 +1,166 @@
+import os
+import sys
+import unittest
+from datetime import datetime, timedelta, timezone
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "POSTGRES_HOST": "localhost",
+ "POSTGRES_PORT": "5432",
+ "POSTGRES_USERNAME": "postgres",
+ "POSTGRES_PASSWORD": "postgres",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.utils.dates import (
+ normalize_date_string,
+ parse_date_string,
+ resolve_relative_date,
+ to_naive_utc,
+)
+
+LOCOMO_TIMESTAMP = "1:56 pm on 8 May, 2023"
+
+
+class ParseDateStringTests(unittest.TestCase):
+ def test_locomo_session_timestamp_parses(self):
+ self.assertEqual(
+ parse_date_string(LOCOMO_TIMESTAMP),
+ datetime(2023, 5, 8, 13, 56),
+ )
+
+ def test_locomo_morning_timestamp_parses(self):
+ self.assertEqual(
+ parse_date_string("10:37 am on 27 June, 2023"),
+ datetime(2023, 6, 27, 10, 37),
+ )
+
+ def test_iso_variants_parse(self):
+ expected = datetime(2023, 5, 8, 13, 56)
+ for value in (
+ "2023-05-08T13:56:00",
+ "2023-05-08T13:56:00.000000",
+ "2023-05-08T13:56:00Z",
+ "2023-05-08T13:56:00+00:00",
+ "2023-05-08 13:56:00",
+ "2023-05-08 13:56:00.000000",
+ "2023-05-08 13:56:00+00:00",
+ ):
+ with self.subTest(value=value):
+ self.assertEqual(parse_date_string(value), expected)
+
+ def test_offset_aware_input_is_converted_to_naive_utc(self):
+ parsed = parse_date_string("2023-05-08T13:56:00+02:00")
+ self.assertEqual(parsed, datetime(2023, 5, 8, 11, 56))
+ self.assertIsNone(parsed.tzinfo)
+
+ def test_parsed_value_never_breaks_subtraction(self):
+ parsed = parse_date_string("2023-05-08T13:56:00-05:00")
+ self.assertIsInstance(datetime.now() - parsed, timedelta)
+
+ def test_ambiguous_numeric_date_is_day_first(self):
+ self.assertEqual(parse_date_string("08/05/2023"), datetime(2023, 5, 8))
+ self.assertEqual(normalize_date_string("08/05/2023"), "08/05/2023")
+
+ def test_day_first_holds_for_unambiguous_high_day(self):
+ self.assertEqual(parse_date_string("25/05/2023"), datetime(2023, 5, 25))
+
+ def test_unparseable_value_returns_none(self):
+ self.assertIsNone(parse_date_string("sometime last summer"))
+
+ def test_previously_supported_formats_still_parse(self):
+ expected = datetime(2023, 5, 8)
+ for value in ("08/05/2023", "2023-05-08", "2023/05/08", "08-05-2023",
+ "May 8, 2023", "May 8 2023", "8 May 2023"):
+ with self.subTest(value=value):
+ self.assertEqual(parse_date_string(value), expected)
+
+
+class NormalizeDateStringTests(unittest.TestCase):
+ def test_output_format_is_day_first_slashed(self):
+ self.assertEqual(normalize_date_string(LOCOMO_TIMESTAMP), "08/05/2023")
+ self.assertEqual(normalize_date_string("2023-05-08T13:56:00Z"), "08/05/2023")
+ self.assertEqual(normalize_date_string("2023-05-08"), "08/05/2023")
+
+ def test_unparseable_value_passes_through_stripped(self):
+ self.assertEqual(
+ normalize_date_string(" sometime last summer "),
+ "sometime last summer",
+ )
+
+
+class ResolveRelativeDateTests(unittest.TestCase):
+ def test_yesterday_against_locomo_timestamp(self):
+ self.assertEqual(
+ resolve_relative_date("yesterday", LOCOMO_TIMESTAMP),
+ "07/05/2023",
+ )
+
+ def test_last_tuesday_against_locomo_timestamp(self):
+ self.assertEqual(
+ resolve_relative_date("last Tuesday", LOCOMO_TIMESTAMP),
+ "02/05/2023",
+ )
+
+ def test_days_ago_against_locomo_timestamp(self):
+ self.assertEqual(
+ resolve_relative_date("3 days ago", LOCOMO_TIMESTAMP),
+ "05/05/2023",
+ )
+
+ def test_weeks_ago_against_locomo_timestamp(self):
+ self.assertEqual(
+ resolve_relative_date("2 weeks ago", LOCOMO_TIMESTAMP),
+ "24/04/2023",
+ )
+
+ def test_unparseable_expression_returns_input_unchanged(self):
+ self.assertEqual(
+ resolve_relative_date("a few months ago", LOCOMO_TIMESTAMP),
+ "a few months ago",
+ )
+
+ def test_unparseable_reference_returns_input_unchanged(self):
+ self.assertEqual(
+ resolve_relative_date("yesterday", "whenever it was"),
+ "yesterday",
+ )
+
+
+class ToNaiveUtcTests(unittest.TestCase):
+ def test_naive_value_is_returned_unchanged(self):
+ value = datetime(2023, 5, 8, 13, 56)
+ self.assertIs(to_naive_utc(value), value)
+
+ def test_aware_value_is_shifted_to_utc(self):
+ value = datetime(2023, 5, 8, 13, 56, tzinfo=timezone(timedelta(hours=2)))
+ self.assertEqual(to_naive_utc(value), datetime(2023, 5, 8, 11, 56))
+
+
+if __name__ == "__main__":
+ unittest.main()