AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""Incremental Notion → isolated wiki-collection sync.
Opt-in (`NOTION_SYNC_ENABLED`). Only pages shared with the Notion
internal integration are visible — that *is* the ACL. Changed pages
(last_edited_time after the per-page record / watermark) are converted
to Markdown, written under ``<company>/docs/notion/``, and re-indexed
into the NOTION Chroma collection keyed by ``notion_page_id``.
That collection is separate from COMPANY: a Notion workspace is
multi-writer, so synced pages are unvetted relative to curated uploads.
The retriever labels them as such and ranks them below company docs.
Heartbeat lifecycle matches ``monitoring.pipeline`` /
``watchlist_research_scan``: bootstrap on boot, run one tick, chain next.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import httpx
from openexecutive.config import Settings, get_settings
from openexecutive.knowledge.loader import DOMAIN_MAP, ingest_text_sync
from openexecutive.knowledge.store import ChromaDBStore
from openexecutive.memory.episodic import insert_scheduled_action
logger = logging.getLogger(__name__)
HEARTBEAT_KIND = "notion_sync_scan"
HEARTBEAT_CHANNEL = "__internal__"
HEARTBEAT_CHANNEL_REF = "notion_sync"
HEARTBEAT_INTENT = "Notion wiki sync — incremental page ingest into isolated collection."
NOTION_VERSION = "2022-06-28"
NOTION_API = "https://api.notion.com/v1"
_MAX_PAGE_CHARS = 200_000
_REQUEST_PAUSE_S = 0.35
MAX_BLOCK_CHILD_PAGES = 20 # 20 × 100-block pages per node — Notion page_size max
_MAX_VISIBLE_PAGES = 2000 # safety valve for reconcile listing, not a Notion API limit
_MAX_CHILD_REQUESTS_PER_PAGE = 80 # hard cap on /blocks/{id}/children calls per page
_CURSOR_RE = re.compile(r"^[A-Za-z0-9_.~+=-]{1,1024}$")
_SLUG_RE = re.compile(r"[^a-z0-9]+")
_UUID_HYPHEN = re.compile(
r"(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
)
_UUID_HEX = re.compile(r"(?i)^[0-9a-f]{32}$")
_PAGE_ID_COMMENT = re.compile(
r"<!--\s*notion_page_id:\s*([0-9a-fA-F-]{32,36})\s*-->"
)
_SILENT_EMPTY_TYPES = {
"paragraph",
"heading_1",
"heading_2",
"heading_3",
"bulleted_list_item",
"numbered_list_item",
"to_do",
"quote",
"toggle",
"callout",
"code",
"divider",
}
def sanitize_notion_id(value: str) -> str | None:
"""Return a hyphenated lowercase Notion UUID, or None if unsafe."""
raw = str(value or "").strip()
if _UUID_HYPHEN.fullmatch(raw):
return raw.lower()
if _UUID_HEX.fullmatch(raw):
h = raw.lower()
return f"{h[:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:]}"
return None
def sanitize_cursor(value: str) -> str | None:
raw = str(value or "").strip()
if _CURSOR_RE.fullmatch(raw):
return raw
return None
def _state_path() -> Path:
settings = get_settings()
return settings.company_profile_path.parent / "notion_sync_state.json"
def _docs_dir() -> Path:
settings = get_settings()
path = settings.company_profile_path.parent / "docs" / "notion"
path.mkdir(parents=True, exist_ok=True)
return path
def load_state() -> dict[str, Any]:
path = _state_path()
if not path.exists():
return {"watermark": None, "pages": {}}
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {"watermark": None, "pages": {}}
if not isinstance(data, dict):
return {"watermark": None, "pages": {}}
data.setdefault("watermark", None)
data.setdefault("pages", {})
if not isinstance(data["pages"], dict):
data["pages"] = {}
return data
def reset_local_state(*, profile_path: Path | None = None) -> None:
"""Drop the on-disk pages/watermark file so the next tick re-ingests.
Call this whenever the Notion collection is wiped (fixture load/reset,
client-slot rebuild) — otherwise leftover page records skip every
still-shared page as 'already current'.
"""
path = (
Path(profile_path).parent / "notion_sync_state.json"
if profile_path is not None
else _state_path()
)
path.unlink(missing_ok=True)
def save_state(state: dict[str, Any]) -> None:
path = _state_path()
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8")
tmp.replace(path)
def rich_text_to_plain(rich: Any) -> str:
if not isinstance(rich, list):
return ""
parts: list[str] = []
for span in rich:
if isinstance(span, dict):
parts.append(str(span.get("plain_text") or ""))
return "".join(parts)
def page_title(page: dict[str, Any]) -> str:
props = page.get("properties") or {}
if isinstance(props, dict):
for value in props.values():
if isinstance(value, dict) and value.get("type") == "title":
title = rich_text_to_plain(value.get("title") or [])
cleaned = _safe_title(title)
if cleaned:
return cleaned
raw = page.get("title")
if isinstance(raw, list):
cleaned = _safe_title(rich_text_to_plain(raw))
if cleaned:
return cleaned
return "Untitled"
def _safe_title(title: str) -> str:
return title.replace("<!--", "").replace("-->", "").strip()
def infer_domain(title: str, extra: str = "") -> str:
hay = f"{title} {extra}".lower()
for key, domain in DOMAIN_MAP.items():
if re.search(rf"\b{re.escape(key)}\b", hay):
return domain
return "general"
def slugify(title: str, page_id: str) -> str:
safe = sanitize_notion_id(page_id) or "invalid"
slug = _SLUG_RE.sub("-", title.lower()).strip("-")[:60] or "page"
short = safe.replace("-", "")[:8]
return f"notion-{short}-{slug}.md"
def block_to_markdown(block: dict[str, Any], list_index: int | None = None) -> str:
btype = str(block.get("type") or "")
raw_body = block.get(btype)
body = raw_body if isinstance(raw_body, dict) else {}
text = rich_text_to_plain(body.get("rich_text") or body.get("text") or [])
if btype == "heading_1":
return f"# {text}" if text else ""
if btype == "heading_2":
return f"## {text}" if text else ""
if btype == "heading_3":
return f"### {text}" if text else ""
if btype == "bulleted_list_item":
return f"- {text}" if text else ""
if btype == "numbered_list_item":
n = list_index if list_index and list_index > 0 else 1
return f"{n}. {text}" if text else ""
if btype == "to_do":
mark = "x" if body.get("checked") else " "
return f"- [{mark}] {text}" if text else ""
if btype == "quote":
return f"> {text}" if text else ""
if btype == "code":
lang = str(body.get("language") or "")
return f"```{lang}\n{text}\n```" if text else ""
if btype == "divider":
return "---"
if btype == "table_row":
cells = body.get("cells") or []
texts = [rich_text_to_plain(c) if isinstance(c, list) else "" for c in cells]
return "| " + " | ".join(texts) + " |" if texts else ""
if btype in {"paragraph", "callout", "toggle"}:
return text
return text
def table_to_markdown(row_blocks: list[dict[str, Any]]) -> str:
rows: list[list[str]] = []
for row in row_blocks:
raw_row = row.get("table_row")
body = raw_row if isinstance(raw_row, dict) else {}
cells = body.get("cells") or []
rows.append(
[rich_text_to_plain(c) if isinstance(c, list) else "" for c in cells]
)
if not rows:
return ""
width = max(len(r) for r in rows)
def _fmt(r: list[str]) -> str:
padded = r + [""] * (width - len(r))
return "| " + " | ".join(padded) + " |"
lines = [_fmt(rows[0])]
lines.append("| " + " | ".join("---" for _ in range(width)) + " |")
lines.extend(_fmt(r) for r in rows[1:])
return "\n".join(lines)
def _headers(api_key: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {api_key}",
"Notion-Version": NOTION_VERSION,
"Content-Type": "application/json",
}
async def _request(
client: httpx.AsyncClient,
method: str,
url: str,
*,
json_body: dict[str, Any] | None = None,
params: dict[str, Any] | None = None,
) -> dict[str, Any]:
response = await client.request(method, url, json=json_body, params=params)
response.raise_for_status()
data = response.json()
if not isinstance(data, dict):
return {}
return data
async def list_shared_pages(
client: httpx.AsyncClient,
*,
max_pages: int,
) -> tuple[list[dict[str, Any]], str | None]:
"""Return (pages, truncated).
``truncated`` is ``None`` for a complete listing, else a short
human-readable cause — callers must treat any non-None value as "more
pages exist that this listing did not return".
"""
pages: list[dict[str, Any]] = []
cursor: str | None = None
truncated: str | None = None
data: dict[str, Any] = {}
max_iters = max(2, (max_pages // 100) + 2)
for _ in range(max_iters):
if len(pages) >= max_pages:
break
body: dict[str, Any] = {
"page_size": min(100, max_pages - len(pages)),
"filter": {"property": "object", "value": "page"},
"sort": {"direction": "descending", "timestamp": "last_edited_time"},
}
if cursor:
safe_cursor = sanitize_cursor(cursor)
if not safe_cursor:
logger.warning("notion_sync: dropping unsafe search cursor")
truncated = "unsafe pagination cursor dropped"
break
body["start_cursor"] = safe_cursor
data = await _request(client, "POST", f"{NOTION_API}/search", json_body=body)
for item in data.get("results") or []:
if isinstance(item, dict) and item.get("object") == "page":
pages.append(item)
if len(pages) >= max_pages:
break
if not data.get("has_more"):
break
cursor = data.get("next_cursor")
if not cursor:
# has_more with no cursor: the listing is incomplete but cannot
# be continued. Mark it truncated so reconciliation is skipped —
# otherwise still-shared pages beyond this point would be purged
# as "missing".
truncated = "Notion returned has_more with no next_cursor"
break
if len(pages) >= max_pages:
truncated = f"listing hit the safety cap of {max_pages} pages"
break
await _sleep()
else:
truncated = "search pagination hit the iteration cap"
logger.warning("notion_sync: search pagination hit iteration cap")
if data.get("has_more") and len(pages) >= max_pages and truncated is None:
truncated = f"listing hit the safety cap of {max_pages} pages"
return pages, truncated
async def _list_child_blocks(
client: httpx.AsyncClient,
block_id: str,
*,
request_budget: list[int],
) -> list[dict[str, Any]]:
safe_id = sanitize_notion_id(block_id)
if not safe_id:
logger.warning("notion_sync: refusing unsafe block id %r", block_id)
return []
blocks: list[dict[str, Any]] = []
cursor: str | None = None
for _page in range(MAX_BLOCK_CHILD_PAGES):
if request_budget[0] <= 0:
logger.warning("notion_sync: per-page request budget exhausted on %s", safe_id)
return blocks
request_budget[0] -= 1
params: dict[str, Any] = {"page_size": 100}
if cursor:
safe_cursor = sanitize_cursor(str(cursor))
if not safe_cursor:
logger.warning("notion_sync: dropping unsafe start_cursor")
break
params["start_cursor"] = safe_cursor
data = await _request(
client,
"GET",
f"{NOTION_API}/blocks/{safe_id}/children",
params=params,
)
for block in data.get("results") or []:
if isinstance(block, dict):
blocks.append(block)
if not data.get("has_more"):
return blocks
cursor = data.get("next_cursor")
if not cursor:
return blocks
await _sleep()
logger.warning(
"notion_sync: block children pagination hit cap %d on %s",
MAX_BLOCK_CHILD_PAGES,
safe_id,
)
return blocks
async def fetch_block_children(
client: httpx.AsyncClient,
block_id: str,
*,
depth: int = 0,
request_budget: list[int] | None = None,
) -> list[str]:
if depth > 8:
return []
if not sanitize_notion_id(block_id):
logger.warning("notion_sync: refusing unsafe block id %r", block_id)
return []
budget = request_budget if request_budget is not None else [_MAX_CHILD_REQUESTS_PER_PAGE]
if budget[0] <= 0:
return []
lines: list[str] = []
dropped: set[str] = set()
list_index = 0
raw_blocks = await _list_child_blocks(client, block_id, request_budget=budget)
for block in raw_blocks:
btype = str(block.get("type") or "")
if btype == "numbered_list_item":
list_index += 1
line = block_to_markdown(block, list_index=list_index)
else:
if btype == "table":
child_id = str(block.get("id") or "")
row_blocks: list[dict[str, Any]] = []
if block.get("has_children") and sanitize_notion_id(child_id):
await _sleep()
row_blocks = await _list_child_blocks(
client, child_id, request_budget=budget
)
line = table_to_markdown(row_blocks)
if not line:
dropped.add("table")
else:
line = block_to_markdown(block)
# Only a block that produces visible output ends a numbered list.
# An empty paragraph between items (common Notion editing artifact)
# must not restart the numbering at 1.
if line:
list_index = 0
if line:
lines.append(line)
elif btype and btype not in _SILENT_EMPTY_TYPES:
dropped.add(btype)
if (
block.get("has_children")
and btype not in {"child_page", "child_database", "table"}
):
child_id = str(block.get("id") or "")
if sanitize_notion_id(child_id):
await _sleep()
lines.extend(
await fetch_block_children(
client, child_id, depth=depth + 1, request_budget=budget
)
)
if dropped:
logger.info(
"notion_sync: dropped block types %s on %s",
sorted(dropped),
block_id,
)
return lines
async def _sleep() -> None:
await asyncio.sleep(_REQUEST_PAUSE_S)
def _page_edited_after(page: dict[str, Any], watermark: str | None) -> bool:
if not watermark:
return True
edited = str(page.get("last_edited_time") or "")
return bool(edited and edited > watermark)
def _page_record(state: dict[str, Any], page_id: str) -> dict[str, Any]:
pages = state.get("pages")
if not isinstance(pages, dict):
return {}
raw = pages.get(page_id)
return raw if isinstance(raw, dict) else {}
def _ingest_page_sync(
page: dict[str, Any],
markdown: str,
store: ChromaDBStore,
) -> int:
page_id = sanitize_notion_id(str(page.get("id") or ""))
if not page_id:
raise ValueError(f"unsafe notion page id: {page.get('id')!r}")
title = page_title(page)
domain = infer_domain(title)
filename = slugify(title, page_id)
dest = _docs_dir() / filename
header = f"<!-- notion_page_id: {page_id} -->\n\n# {title}\n\n"
body = (header + markdown).strip()[:_MAX_PAGE_CHARS]
dest.write_text(body + "\n", encoding="utf-8")
_remove_stale_page_files(page_id, keep=dest)
store.delete_documents(
ChromaDBStore.NOTION_COLLECTION, {"notion_page_id": page_id}
)
store.delete_documents(
ChromaDBStore.COMPANY_COLLECTION, {"notion_page_id": page_id}
)
return ingest_text_sync(
body,
store,
source_name=f"notion/{filename}",
domain=domain,
collection=ChromaDBStore.NOTION_COLLECTION,
extra_metadata={
"notion_page_id": page_id,
"type": "notion",
},
)
async def ingest_page(
page: dict[str, Any],
markdown: str,
store: ChromaDBStore,
) -> int:
return await asyncio.to_thread(_ingest_page_sync, page, markdown, store)
def _safe_filename(name: str) -> str | None:
candidate = Path(str(name)).name
if candidate.startswith("notion-") and candidate.endswith(".md"):
return candidate
return None
def _build_file_index() -> dict[str, list[Path]]:
"""One directory pass: map each synced page id to its on-disk file(s).
Reading every file head is the unavoidable cost of orphan detection;
building the index once per operation keeps ``purge_page`` /
``reconcile_missing_pages`` from re-scanning the directory per page.
"""
index: dict[str, list[Path]] = {}
for path in _docs_dir().glob("notion-*.md"):
try:
head = path.read_text(encoding="utf-8", errors="replace")[:4000]
except OSError:
continue
found = _PAGE_ID_COMMENT.search(head)
file_id = sanitize_notion_id(found.group(1)) if found else None
if file_id:
index.setdefault(file_id, []).append(path)
return index
def purge_page(
page_id: str,
store: ChromaDBStore,
state: dict[str, Any] | None = None,
file_index: dict[str, list[Path]] | None = None,
) -> bool:
"""Remove one synced page's file, chunks, and state record.
``file_index`` (from :func:`_build_file_index`) avoids a per-call
directory scan; direct callers may omit it and one is built here.
"""
pid = sanitize_notion_id(page_id)
if not pid:
logger.warning("notion_sync: refuse to purge unsafe page id %r", page_id)
return False
meta: dict[str, Any] = {}
pages: dict[str, Any] | None = None
if state is not None:
raw_pages = state.setdefault("pages", {})
if not isinstance(raw_pages, dict):
state["pages"] = {}
raw_pages = state["pages"]
pages = raw_pages
raw = pages.get(pid)
if raw is None:
raw = pages.get(page_id)
if isinstance(raw, dict):
meta = raw
filename = _safe_filename(str(meta.get("filename") or ""))
docs = _docs_dir()
store.delete_documents(ChromaDBStore.NOTION_COLLECTION, {"notion_page_id": pid})
store.delete_documents(ChromaDBStore.COMPANY_COLLECTION, {"notion_page_id": pid})
if filename:
(docs / filename).unlink(missing_ok=True)
index = file_index if file_index is not None else _build_file_index()
for path in index.get(pid, []):
path.unlink(missing_ok=True)
if pages is not None:
pages.pop(pid, None)
pages.pop(page_id, None)
logger.info("notion_sync: purged page %s", pid)
return True
def reconcile_missing_pages(
visible_ids: set[str],
store: ChromaDBStore,
state: dict[str, Any],
) -> int:
"""Purge pages (and orphan files) the integration no longer sees.
Blocking file and Chroma I/O throughout — callers on the event loop
must run this via ``asyncio.to_thread`` (the sync tick does).
"""
pages = state.setdefault("pages", {})
if not isinstance(pages, dict):
state["pages"] = {}
pages = state["pages"]
file_index = _build_file_index()
stale = [pid for pid in list(pages) if sanitize_notion_id(str(pid)) not in visible_ids]
purged = 0
purged_ids: set[str] = set()
for pid in stale:
if purge_page(str(pid), store, state, file_index=file_index):
purged += 1
safe = sanitize_notion_id(str(pid))
if safe:
purged_ids.add(safe)
# Orphan files: on disk with a page-id comment but absent from state.
for file_id in list(file_index):
if (
file_id not in visible_ids
and file_id not in purged_ids
and purge_page(file_id, store, state, file_index=file_index)
):
purged += 1
return purged
def _remove_stale_page_files(page_id: str, keep: Path) -> None:
keep_resolved = keep.resolve()
for path in _docs_dir().glob("notion-*.md"):
try:
if path.resolve() == keep_resolved:
continue
head = path.read_text(encoding="utf-8", errors="replace")[:4000]
except OSError:
continue
found = _PAGE_ID_COMMENT.search(head)
if found and sanitize_notion_id(found.group(1)) == page_id:
path.unlink(missing_ok=True)
def _clear_legacy_company_notion(store: ChromaDBStore) -> None:
store.delete_documents(ChromaDBStore.COMPANY_COLLECTION, {"type": "notion"})
@dataclass
class _TickFetch:
"""Everything the network phase of a tick learned, ready to apply locally."""
visible_ids: set[str] = field(default_factory=set)
truncated: str | None = None
# (page, page_id, last_edited, markdown) per successfully fetched page.
fetched: list[tuple[dict[str, Any], str, str, str]] = field(default_factory=list)
# last_edited times of pages left unsynced (fetch failure / cap overflow),
# which pin the watermark below them.
unresolved_times: list[str] = field(default_factory=list)
def _note_reconcile_skip(state: dict[str, Any], cause: str) -> None:
"""Record and log one more consecutive tick without reconciliation.
Reconciliation is what purges pages whose sharing was revoked, so a
persistent skip streak means revoked content is lingering locally —
surface the streak length instead of logging each skip in isolation.
"""
raw = state.get("reconcile_skips")
skips = (raw if isinstance(raw, int) else 0) + 1
state["reconcile_skips"] = skips
logger.warning(
"notion_sync: skipping reconciliation — %s "
"(%d consecutive skip(s); revoked pages are not purged until "
"reconciliation runs)",
cause,
skips,
)
async def _fetch_tick(
*,
client: httpx.AsyncClient,
settings: Settings,
state: dict[str, Any],
reconcile_only: bool,
stats: dict[str, int],
) -> _TickFetch:
"""Network phase: list visible pages and download dirty page content.
Reads ``state`` only to decide which pages are already current; all
state mutation happens later in :func:`_apply_tick`.
"""
fetch = _TickFetch()
pages, fetch.truncated = await list_shared_pages(
client, max_pages=_MAX_VISIBLE_PAGES
)
stats["seen"] = len(pages)
dirty: list[dict[str, Any]] = []
for page in pages:
page_id = sanitize_notion_id(str(page.get("id") or ""))
if not page_id:
logger.warning(
"notion_sync: skipping page with unsafe id %r", page.get("id")
)
stats["failed"] += 1
continue
fetch.visible_ids.add(page_id)
edited = str(page.get("last_edited_time") or "")
recorded = _page_record(state, page_id)
if recorded.get("last_edited") == edited:
stats["skipped"] += 1
continue
dirty.append(page)
if reconcile_only:
return fetch
ingest_cap = max(0, settings.notion_max_pages_per_scan)
dirty.sort(
key=lambda p: str(p.get("last_edited_time") or ""),
reverse=True,
)
overflow = dirty[ingest_cap:]
to_ingest = dirty[:ingest_cap]
if overflow:
stats["capped"] = len(overflow)
logger.warning(
"notion_sync: %d page(s) need ingest, cap is %d — "
"overflow will retry next tick (watermark not advanced past them)",
len(dirty),
ingest_cap,
)
fetch.unresolved_times = [
str(p.get("last_edited_time") or "")
for p in overflow
if p.get("last_edited_time")
]
for page in to_ingest:
page_id = sanitize_notion_id(str(page.get("id") or ""))
if not page_id:
stats["failed"] += 1
continue
edited = str(page.get("last_edited_time") or "")
try:
await _sleep()
lines = await fetch_block_children(client, page_id)
except Exception:
stats["failed"] += 1
if edited:
fetch.unresolved_times.append(edited)
logger.exception("notion_sync: failed to fetch page %s", page_id)
continue
fetch.fetched.append((page, page_id, edited, "\n\n".join(lines)))
return fetch
async def _apply_tick(
*,
store: ChromaDBStore,
fetch: _TickFetch,
now: datetime | None,
reconcile_only: bool,
stats: dict[str, int],
) -> None:
"""Local write phase — the caller must hold ``_FIXTURE_OP_LOCK``.
Nothing here touches the network: it is bounded file, state, and
Chroma work, so the lock hold stays short even for a large wiki.
"""
# Reload instead of reusing the pre-fetch snapshot: a fixture load or
# reset may have replaced the state file while the network phase ran,
# and saving a stale snapshot would resurrect purged page records.
state = load_state()
watermark = state.get("watermark") if isinstance(state.get("watermark"), str) else None
await asyncio.to_thread(_clear_legacy_company_notion, store)
known = state.get("pages") if isinstance(state.get("pages"), dict) else {}
if fetch.truncated:
_note_reconcile_skip(
state,
f"page listing incomplete ({fetch.truncated}), "
"so unseen pages must not be purged as missing",
)
elif not fetch.visible_ids and known:
_note_reconcile_skip(
state,
f"search returned 0 pages while {len(known)} are on record "
"(refusing a mass purge on a blank listing)",
)
else:
# File-head scanning plus Chroma deletes — keep it off the event
# loop so a large or bulk-unshared wiki cannot stall the API.
stats["purged"] = await asyncio.to_thread(
reconcile_missing_pages, fetch.visible_ids, store, state
)
state["reconcile_skips"] = 0
if not reconcile_only:
synced_at = (now or datetime.now(UTC)).isoformat()
unresolved_times = list(fetch.unresolved_times)
for page, page_id, edited, markdown in fetch.fetched:
try:
chunks = await ingest_page(page, markdown, store)
filename = slugify(page_title(page), page_id)
record: dict[str, Any] = {
"last_edited": edited,
"title": page_title(page),
"filename": filename,
"synced_at": synced_at,
}
url = page.get("url")
if isinstance(url, str) and url.startswith("https://"):
record["url"] = url
state.setdefault("pages", {})[page_id] = record
stats["updated"] += 1
logger.info(
"notion_sync: indexed %s (%d chunks)", page_title(page), chunks
)
except Exception:
stats["failed"] += 1
if edited:
unresolved_times.append(edited)
logger.exception("notion_sync: failed page %s", page_id)
if unresolved_times:
# Leave watermark strictly below the earliest unresolved edit
# so those pages stay eligible. Successfully recorded pages
# are skipped via the pages dict, not the watermark.
logger.info(
"notion_sync: watermark held at %s (%d unresolved page(s))",
watermark,
len(unresolved_times),
)
else:
recorded_times = [
str(rec.get("last_edited") or "")
for rec in (state.get("pages") or {}).values()
if isinstance(rec, dict) and rec.get("last_edited")
]
if recorded_times:
candidate = max(recorded_times)
if watermark is None or candidate > watermark:
state["watermark"] = candidate
state["last_run"] = (now or datetime.now(UTC)).isoformat()
# A tick that skipped some pages still succeeded, but the knowledge page
# should say so rather than show a clean sync.
if stats["failed"]:
state["last_error"] = _partial_failure_message(stats["failed"])
else:
state.pop("last_error", None)
save_state(state)
def _partial_failure_message(failed: int) -> str:
return (
f"{failed} page{'' if failed == 1 else 's'} could not be synced. "
"Check the server logs."
)
# One tick at a time per process — see the matching lock in drive_sync.
_RUN_LOCK = asyncio.Lock()
# When the last tick in this process ended, failed or not. ``last_run`` is
# stamped only on success and with the tick's start time, so a manual-sync
# cooldown keyed on it alone lets a failing or slow source be re-run
# back-to-back.
_last_finished_at: datetime | None = None
def is_syncing() -> bool:
return _RUN_LOCK.locked()
def last_finished_at() -> datetime | None:
return _last_finished_at
def _record_error(message: str) -> None:
"""Keep a short, user-facing reason the last tick failed (never a trace)."""
try:
state = load_state()
state["last_error"] = message
save_state(state)
except Exception:
logger.exception("notion_sync: could not record the last error")
async def run_notion_sync(
*,
store: ChromaDBStore | None = None,
client: httpx.AsyncClient | None = None,
now: datetime | None = None,
reconcile_only: bool = False,
) -> dict[str, int]:
"""One sync tick. Returns counts: seen / updated / skipped / failed / purged / capped
(plus ``busy`` when another tick is already running in this process)."""
if _RUN_LOCK.locked():
logger.info("notion_sync: a tick is already running — skipping")
return {"busy": 1}
global _last_finished_at
async with _RUN_LOCK:
try:
return await _run_notion_sync_locked(
store=store, client=client, now=now, reconcile_only=reconcile_only
)
except Exception:
_record_error("The last sync failed unexpectedly. Check the server logs.")
raise
finally:
_last_finished_at = datetime.now(UTC)
async def _run_notion_sync_locked(
*,
store: ChromaDBStore | None,
client: httpx.AsyncClient | None,
now: datetime | None,
reconcile_only: bool,
) -> dict[str, int]:
settings = get_settings()
stats = {
"seen": 0,
"updated": 0,
"skipped": 0,
"failed": 0,
"purged": 0,
"capped": 0,
}
if not settings.notion_sync_enabled:
return stats
api_key = settings.notion_api_key
if not api_key:
logger.warning("notion_sync: enabled but NOTION_API_KEY is empty — skipping")
return stats
# A sync tick writes the active client's docs, vector collections, and
# sync-state file, so its writes must not interleave with an in-process
# fixture load / client rotation — those swap that state wholesale and a
# mid-rotation tick would write one client's wiki into another's
# collection. Holding the destructive-op lock across the whole tick
# would let slow Notion I/O block /fixtures/* and /clients/* (and, via
# the rotation pause marker, the entire scheduler), so instead: skip
# the tick when a destructive op is already running, fetch from Notion
# unlocked, and take the lock only for the local write phase. A CLI
# invocation is a separate process this lock cannot see — that
# cross-process race is a known, accepted limitation.
from openexecutive.cli.fixture_loader import _FIXTURE_OP_LOCK
from openexecutive.clients.slots import get_active_client
if _FIXTURE_OP_LOCK.locked():
logger.info(
"notion_sync: fixture/rotation operation in progress — "
"skipping this tick, will retry on the next interval"
)
return stats
if store is None:
store = ChromaDBStore(persist_directory=settings.vector_store_path)
generation = get_active_client(settings)
own_client = client is None
if client is None:
client = httpx.AsyncClient(headers=_headers(api_key), timeout=60.0)
try:
fetch = await _fetch_tick(
client=client,
settings=settings,
state=load_state(),
reconcile_only=reconcile_only,
stats=stats,
)
async with _FIXTURE_OP_LOCK:
if get_active_client(settings) != generation:
logger.warning(
"notion_sync: active client changed while fetching — "
"discarding this tick's results"
)
return stats
await _apply_tick(
store=store,
fetch=fetch,
now=now,
reconcile_only=reconcile_only,
stats=stats,
)
finally:
if own_client:
await client.aclose()
logger.info("notion_sync: %s", stats)
return stats
def purge_all_synced(store: ChromaDBStore, state: dict[str, Any] | None = None) -> int:
"""Remove every locally synced Notion page (files + chunks + state)."""
current = state if state is not None else load_state()
raw_pages = current.get("pages")
pages = raw_pages if isinstance(raw_pages, dict) else {}
ids = list(pages)
file_index = _build_file_index()
purged = 0
purged_ids: set[str] = set()
for pid in ids:
if purge_page(str(pid), store, current, file_index=file_index):
purged += 1
safe = sanitize_notion_id(str(pid))
if safe:
purged_ids.add(safe)
for file_id in file_index:
if file_id not in purged_ids and purge_page(
file_id, store, current, file_index=file_index
):
purged += 1
indexed_paths = {p for paths in file_index.values() for p in paths}
for path in _docs_dir().glob("notion-*.md"):
# Files with no readable page-id comment are junk — remove them.
if path not in indexed_paths:
path.unlink(missing_ok=True)
purged += 1
store.delete_notion_docs()
current["pages"] = {}
current["watermark"] = None
save_state(current)
return purged
def _heartbeat_pending(db_path: Path | None = None) -> bool:
from openexecutive.memory.episodic import _get_conn, _resolve_db_path
resolved = _resolve_db_path(db_path)
if not resolved.exists():
return False
with _get_conn(resolved) as conn:
row = conn.execute(
"SELECT 1 FROM scheduled_actions "
"WHERE kind = ? AND status IN ('pending', 'running') LIMIT 1",
(HEARTBEAT_KIND,),
).fetchone()
return row is not None
def bootstrap_notion_sync_scan(db_path: Path | None = None) -> int | None:
if _heartbeat_pending(db_path):
return None
run_at = datetime.now(UTC) + timedelta(minutes=1)
try:
action_id = insert_scheduled_action(
run_at=run_at.isoformat(),
channel=HEARTBEAT_CHANNEL,
channel_ref=HEARTBEAT_CHANNEL_REF,
intent_text=HEARTBEAT_INTENT,
kind=HEARTBEAT_KIND,
db_path=db_path,
)
logger.info("notion_sync.bootstrap: heartbeat at %s (id=%d)", run_at.isoformat(), action_id)
return action_id
except Exception:
logger.exception("notion_sync.bootstrap: failed to enqueue heartbeat")
return None
def enqueue_next_notion_sync_scan(
*,
after: datetime | None = None,
db_path: Path | None = None,
) -> int | None:
settings = get_settings()
base = (after or datetime.now(UTC)).astimezone(UTC)
run_at = base + timedelta(minutes=settings.notion_sync_interval_minutes)
try:
action_id = insert_scheduled_action(
run_at=run_at.isoformat(),
channel=HEARTBEAT_CHANNEL,
channel_ref=HEARTBEAT_CHANNEL_REF,
intent_text=HEARTBEAT_INTENT,
kind=HEARTBEAT_KIND,
db_path=db_path,
)
logger.info("notion_sync.enqueue_next: next scan at %s (id=%d)", run_at.isoformat(), action_id)
return action_id
except Exception:
logger.exception("notion_sync.enqueue_next: insert failed")
return None
def list_synced_pages() -> dict[str, Any]:
"""What the knowledge page shows for Notion: one row per page on record.
Mirrors ``drive_sync.list_synced_files``. Pages recorded before
``synced_at`` / ``url`` were kept per page fall back to the tick time and
no link.
"""
state = load_state()
last_run = state.get("last_run") if isinstance(state.get("last_run"), str) else None
raw_pages = state.get("pages")
pages: list[dict[str, Any]] = []
for pid, rec in (raw_pages if isinstance(raw_pages, dict) else {}).items():
safe = sanitize_notion_id(str(pid))
if not safe or not isinstance(rec, dict):
continue
filename = _safe_filename(str(rec.get("filename") or ""))
url = rec.get("url")
pages.append(
{
"id": safe,
"name": str(rec.get("title") or filename or safe),
"url": url if isinstance(url, str) and url.startswith("https://") else None,
"modified_at": rec.get("last_edited")
if isinstance(rec.get("last_edited"), str)
else None,
"synced_at": rec.get("synced_at")
if isinstance(rec.get("synced_at"), str)
else last_run,
"indexed": filename is not None,
}
)
pages.sort(key=lambda p: p["name"].lower())
error = state.get("last_error")
return {
"last_run": last_run,
"last_error": error if isinstance(error, str) else None,
"files": pages,
}
def read_synced_page(page_id: str) -> dict[str, Any] | None:
"""The stored Markdown of one synced page, or None when it is unknown.
The path comes from the state record for a sanitized id, never from the
caller, and must be a ``notion-*.md`` name directly under ``docs/notion/``.
"""
safe = sanitize_notion_id(page_id)
if not safe:
return None
raw_pages = load_state().get("pages")
rec = raw_pages.get(safe) if isinstance(raw_pages, dict) else None
if not isinstance(rec, dict):
return None
filename = _safe_filename(str(rec.get("filename") or ""))
if filename is None:
return None
try:
text = (_docs_dir() / filename).read_text(encoding="utf-8", errors="replace")
except OSError:
return None
url = rec.get("url")
return {
"id": safe,
"name": str(rec.get("title") or filename),
"url": url if isinstance(url, str) and url.startswith("https://") else None,
"content": _PAGE_ID_COMMENT.sub("", text, count=1).lstrip(),
}