AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""Insert-time validation + normalization of watchlist targets.
The watchlist is largely populated by an LLM (the ``executive_research``
workflow) proposing ``rss`` feed URLs it never verified — historically
the great majority were dead (404 / 403 / not-actually-a-feed), producing
a watchlist full of rows that fetch nothing and a permanently-empty
briefing.
This module validates an ``rss`` target at insert time: it must fetch and
parse as a feed with entries. A target the server *answers* but that
isn't a feed is either:
- converted to a ``page_watch`` when the page the feed check already
fetched yields readable text (a news or pricing page with no feed), or
- rejected (:class:`WatchlistTargetError`) when it does not,
so a dead ``rss`` row never reaches the scan loop. A transient / network
blip passes the target through unchanged rather than blocking a
legitimate add on a hiccup. Every non-``rss`` type returns unchanged with
no network I/O.
"""
from __future__ import annotations
import logging
from typing import Any, Literal
import feedparser
import httpx
from openexecutive.config import get_settings
from openexecutive.monitoring.models import (
SOURCE_KIND_PAGE_WATCH,
SOURCE_KIND_RSS,
)
from openexecutive.monitoring.sources._http import (
FetchOverflowError,
fetch_bounded,
validate_target_url,
)
from openexecutive.monitoring.sources.page_watch import html_to_text
logger = logging.getLogger(__name__)
# A non-feed page becomes a page_watch only when its visible text is at least
# this long. Rules out empty shells, bare error strings and binary bodies that
# happen to decode to a few characters; a real article, pricing or news page
# clears it easily.
_MIN_PAGE_TEXT_CHARS = 200
# Only the head of the body is reduced for that check: 200 visible
# characters never need more, and the insert path runs on the request
# thread, so the work is bounded whatever the server sends.
_READABILITY_SCAN_BYTES = 262_144
class WatchlistTargetError(Exception):
"""A watchlist target failed insert-time validation (definitively bad)."""
def __init__(self, reason: str) -> None:
super().__init__(reason)
self.reason = reason
async def validate_and_normalize_target(
signal_type: str, target: str, config: dict[str, Any],
) -> tuple[str, str, dict[str, Any]]:
"""Validate ``target`` for ``signal_type``; may convert the signal type.
Returns the ``(signal_type, target, config)`` to insert — converted to
``page_watch`` when a non-feed ``rss`` target is a readable page. Raises
:class:`WatchlistTargetError` when an ``rss`` target is definitively
bad and can't be salvaged. Only ``rss`` is validated; every other type
is returned unchanged with no network I/O.
"""
if signal_type != SOURCE_KIND_RSS:
return signal_type, target, config
verdict, body = await _feed_check(target)
if verdict == "feed":
return signal_type, target, config
if verdict == "unknown":
# Transient fetch failure — inserting as-is beats rejecting a feed
# that's merely unreachable right now; the scan loop retries.
logger.info(
"watchlist validate: %r unreachable now — inserting rss unverified",
target,
)
return signal_type, target, config
if verdict == "blocked":
# SSRF-rejected / unparseable / non-public. Reject outright.
raise WatchlistTargetError(
f"{target!r} is not a fetchable public URL "
"(blocked by the SSRF guard or unparseable)."
)
# verdict == "not_feed": a public server answered, but it is not a feed.
# The body the feed check already fetched (same SSRF guard, same byte
# cap the page_watch adapter uses) decides whether the page is worth
# watching: readable text → page_watch; nothing readable → reject.
if body is not None and _is_readable_page(body):
new_config = dict(config)
# page_watch reads config["label"]; the rss row used "feed_label".
label = new_config.pop("feed_label", None)
if label and "label" not in new_config:
new_config["label"] = label
logger.info(
"watchlist validate: %r is not a feed but is a readable page — "
"converting rss → page_watch", target,
)
return SOURCE_KIND_PAGE_WATCH, target, new_config
raise WatchlistTargetError(
f"{target!r} is not a valid RSS/Atom feed and yields no readable "
"page text. Fix the feed URL, or add it as signal_type='page_watch'."
)
def _is_readable_page(body: bytes) -> bool:
"""Whether a fetched body is a text page with enough visible content to
watch. A NUL byte in the head marks a binary body (PDF, image, archive);
the text threshold rejects shells and one-line error pages."""
if b"\x00" in body[:4096]:
return False
return len(html_to_text(body[:_READABILITY_SCAN_BYTES])) >= _MIN_PAGE_TEXT_CHARS
async def _feed_check(
target: str,
) -> tuple[Literal["feed", "not_feed", "blocked", "unknown"], bytes | None]:
"""Classify ``target``; also return the fetched body when there is one.
``feed`` — feedparser recognized it as a feed.
``not_feed`` — a public server answered, but it isn't a feed (the body
is returned so the caller can decide whether it is a readable page;
``None`` when the server answered with an error or an oversized body).
``blocked`` — SSRF-rejected / unparseable / non-public URL (never fetched).
``unknown`` — transient fetch failure; caller should not penalize it.
"""
ok, _reason = validate_target_url(target)
if not ok:
return "blocked", None
max_bytes = get_settings().external_monitor_max_fetch_bytes
try:
body = await fetch_bounded(target, max_bytes)
except FetchOverflowError:
return "not_feed", None # too big to be a sane feed or page
except httpx.HTTPStatusError:
return "not_feed", None # server answered 4xx/5xx → definitively bad
except (httpx.HTTPError, httpx.InvalidURL):
return "unknown", None # connection / timeout — transient
# feedparser sets `version` (e.g. "rss20", "atom10") for anything it
# recognizes as a feed — INCLUDING a valid but currently-empty feed. Use
# it rather than `.entries` so a brand-new / quiet feed isn't misread as a
# non-feed page and wrongly converted or rejected. Use dict `.get` (not
# attribute access): an empty body yields a FeedParserDict with no
# `version` key at all, and `parsed.version` would AttributeError.
parsed = feedparser.parse(body)
return ("feed", body) if parsed.get("version") else ("not_feed", body)
__all__ = ["WatchlistTargetError", "validate_and_normalize_target"]