:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import re
import base64
import binascii
from collections.abc import Callable
from datetime import UTC, datetime
from email.header import decode_header, make_header
from email.utils import getaddresses, parseaddr
from html import unescape
from typing import Any
from django.utils import timezone
from django.utils.html import strip_tags
import requests
from bs4 import BeautifulSoup, NavigableString, PageElement, Tag
from posthog.dataclasses import frozen
from posthog.egress.google_workspace import google_workspace_request
from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, OauthIntegration
from posthog.models.organization import OrganizationMembership
from posthog.models.team import Team
from posthog.models.user import User
from products.conversations.backend.models import EmailChannel, EmailChannelKind, EmailThreadMessageDirection
from products.conversations.backend.services.email_thread_ingestion import (
EmailAddress,
ParsedEmail,
ingest_customer_email,
)
GMAIL_READONLY_SCOPE = "https://www.googleapis.com/auth/gmail.readonly"
GMAIL_HISTORY_ID_CONFIG_KEY = "gmail_history_id"
GMAIL_LAST_SYNCED_AT_CONFIG_KEY = "gmail_last_synced_at"
GMAIL_HISTORY_START_ID_CONFIG_KEY = "gmail_history_start_id"
GMAIL_HISTORY_PAGE_TOKEN_CONFIG_KEY = "gmail_history_page_token"
GMAIL_HISTORY_TARGET_ID_CONFIG_KEY = "gmail_history_target_id"
GMAIL_PENDING_MESSAGE_IDS_CONFIG_KEY = "gmail_pending_message_ids"
GMAIL_API_BASE_URL = "https://gmail.googleapis.com/gmail/v1/users/me"
INITIAL_IMPORT_QUERY = "{in:inbox in:sent} newer_than:30d"
INITIAL_IMPORT_LIMIT = 100
_HTML_PARAGRAPH_TAGS = frozenset({"blockquote", "h1", "h2", "h3", "h4", "h5", "h6", "ol", "p", "table", "ul"})
_HTML_LINE_TAGS = frozenset({"div", "li", "tr"})
_HTML_SKIPPED_TAGS = frozenset({"head", "script", "style", "title"})
_HTML_WHITESPACE_RE = re.compile(r"\s+")
_BLANK_LINES_RE = re.compile(r"\n{3,}")
_PREFORMATTED_PLACEHOLDER_RE = re.compile(r"\x00(\d+)\x00")
BACKFILL_PAGE_SIZE = 5
HISTORY_PAGE_SIZE = 100
HISTORY_MESSAGE_BATCH_SIZE = 100
MAX_ATTACHMENT_BACKED_BODY_PARTS = 4
_MESSAGE_ID_RE = re.compile(r"<[^<>\s]+>")
class GmailSyncError(Exception):
pass
class _GmailBodyPartLimitExceeded(Exception):
pass
@frozen
class _EmailBodies:
plain: str
html: str
def integration_has_gmail_scope(integration: IntegrationRecord) -> bool:
scopes = integration.config.get("scope") or ""
return GMAIL_READONLY_SCOPE in scopes.split()
def can_sync_gmail_integration(integration_id: int, team_id: int) -> bool:
try:
integration = IntegrationManager.google_calendar.oauth.get(team_id, integration_id)
except IntegrationNotFound:
return False
if not integration_has_gmail_scope(integration) or not _has_active_owner(integration):
return False
email = str(integration.config.get("email") or "").strip()
return "@" in email
def sync_gmail_integration(integration_id: int, team_id: int) -> None:
integration = IntegrationManager.google_calendar.oauth.get(team_id, integration_id)
if not integration_has_gmail_scope(integration) or not _has_active_owner(integration):
return
access_token = _get_fresh_access_token(integration)
channel = _email_channel(integration)
internal_emails = _organization_member_emails(integration)
history_id = integration.config.get(GMAIL_HISTORY_ID_CONFIG_KEY)
if history_id:
next_history_id = _sync_history(
integration=integration,
channel=channel,
access_token=access_token,
start_history_id=str(history_id),
internal_emails=internal_emails,
)
else:
next_history_id = _initial_sync(
integration=integration,
channel=channel,
access_token=access_token,
internal_emails=internal_emails,
)
if next_history_id is None:
return
IntegrationManager.google_calendar.oauth.update(
integration.team_id,
integration.id,
config_patch={
GMAIL_HISTORY_ID_CONFIG_KEY: next_history_id,
GMAIL_LAST_SYNCED_AT_CONFIG_KEY: timezone.now().isoformat(),
},
)
def sync_gmail_backfill_batch(
integration_id: int,
team_id: int,
*,
start_at: datetime,
end_at: datetime,
page_token: str | None = None,
) -> tuple[str | None, int]:
integration = IntegrationManager.google_calendar.oauth.get(team_id, integration_id)
if not integration_has_gmail_scope(integration) or not _has_active_owner(integration):
raise GmailSyncError(f"Integration {integration.id} cannot sync Gmail")
access_token = _get_fresh_access_token(integration)
channel = _email_channel(integration)
params = {
"q": f"{{in:inbox in:sent}} after:{int(start_at.timestamp()) - 1} before:{int(end_at.timestamp())}",
"maxResults": BACKFILL_PAGE_SIZE,
}
if page_token:
params["pageToken"] = page_token
payload = _get_json(
integration=integration,
access_token=access_token,
url=f"{GMAIL_API_BASE_URL}/messages",
endpoint="/gmail/v1/users/me/messages",
params=params,
)
message_ids = [str(message["id"]) for message in payload.get("messages", []) if message.get("id")]
_ingest_message_ids(
integration=integration,
channel=channel,
access_token=access_token,
message_ids=message_ids,
internal_emails=_organization_member_emails(integration),
)
next_page_token = str(payload.get("nextPageToken") or "") or None
return next_page_token, len(message_ids)
def _owner(integration: IntegrationRecord) -> User | None:
if integration.created_by_id is None:
return None
return User.objects.filter(id=integration.created_by_id).first()
def _has_active_owner(integration: IntegrationRecord) -> bool:
owner = _owner(integration)
return bool(owner and owner.is_active and owner.teams.filter(id=integration.team_id).exists())
def _get_fresh_access_token(integration: IntegrationRecord) -> str:
oauth = OauthIntegration(integration)
if integration.errors != ERROR_TOKEN_REFRESH_FAILED and oauth.access_token_expired():
oauth.refresh_access_token()
if integration.errors == ERROR_TOKEN_REFRESH_FAILED:
raise GmailSyncError(f"Token refresh failed for integration {integration.id}")
token = integration.access_token
if not token:
raise GmailSyncError(f"Integration {integration.id} has no access token")
return token
def _email_channel(integration: IntegrationRecord) -> EmailChannel:
owner = _owner(integration)
email = str(integration.config.get("email") or "").strip().lower()
if owner is None or not email or "@" not in email:
raise GmailSyncError(f"Integration {integration.id} has no mailbox owner")
name = " ".join(part for part in (owner.first_name, owner.last_name) if part).strip() or email
return EmailChannel(
team=Team.objects.get(id=integration.team_id),
kind=EmailChannelKind.CUSTOMER_COMMUNICATION,
owner=owner,
inbound_token="",
from_email=email,
from_name=name,
domain=email.rsplit("@", 1)[1],
domain_verified=False,
dns_records={},
is_default=False,
)
def _organization_member_emails(integration: IntegrationRecord) -> set[str]:
return {
email.lower()
for email in OrganizationMembership.objects.filter(
organization_id=Team.objects.values_list("organization_id", flat=True).get(id=integration.team_id),
user__is_active=True,
).values_list("user__email", flat=True)
}
def _initial_sync(
*,
integration: IntegrationRecord,
channel: EmailChannel,
access_token: str,
internal_emails: set[str],
) -> str:
profile = _get_json(
integration=integration,
access_token=access_token,
url=f"{GMAIL_API_BASE_URL}/profile",
endpoint="/gmail/v1/users/me/profile",
)
history_id = str(profile.get("historyId") or "")
if not history_id:
raise GmailSyncError("Gmail profile did not include a history cursor")
payload = _get_json(
integration=integration,
access_token=access_token,
url=f"{GMAIL_API_BASE_URL}/messages",
endpoint="/gmail/v1/users/me/messages",
params={"q": INITIAL_IMPORT_QUERY, "maxResults": INITIAL_IMPORT_LIMIT},
)
_ingest_message_ids(
integration=integration,
channel=channel,
access_token=access_token,
message_ids=[str(message["id"]) for message in payload.get("messages", []) if message.get("id")],
internal_emails=internal_emails,
)
return history_id
def _sync_history(
*,
integration: IntegrationRecord,
channel: EmailChannel,
access_token: str,
start_history_id: str,
internal_emails: set[str],
) -> str | None:
processed_messages = 0
while processed_messages < HISTORY_MESSAGE_BATCH_SIZE:
config = IntegrationManager.google_calendar.oauth.get(integration.team_id, integration.id).config or {}
progress_start_id = str(config.get(GMAIL_HISTORY_START_ID_CONFIG_KEY) or start_history_id)
page_token = str(config.get(GMAIL_HISTORY_PAGE_TOKEN_CONFIG_KEY) or "") or None
target_history_id = str(config.get(GMAIL_HISTORY_TARGET_ID_CONFIG_KEY) or "") or None
pending_message_ids = [str(message_id) for message_id in config.get(GMAIL_PENDING_MESSAGE_IDS_CONFIG_KEY) or []]
if pending_message_ids:
_ingest_message_id(
integration=integration,
channel=channel,
access_token=access_token,
gmail_message_id=pending_message_ids[0],
internal_emails=internal_emails,
)
_save_history_progress(
integration,
start_history_id=progress_start_id,
page_token=page_token,
target_history_id=target_history_id,
pending_message_ids=pending_message_ids[1:],
)
processed_messages += 1
continue
if target_history_id and page_token is None:
_clear_history_progress(integration)
return target_history_id
params = {
"startHistoryId": progress_start_id,
"historyTypes": "messageAdded",
"maxResults": HISTORY_PAGE_SIZE,
}
if page_token:
params["pageToken"] = page_token
response = _request(
integration=integration,
access_token=access_token,
url=f"{GMAIL_API_BASE_URL}/history",
endpoint="/gmail/v1/users/me/history",
params=params,
)
if response.status_code == 404:
_clear_history_progress(integration)
return _initial_sync(
integration=integration,
channel=channel,
access_token=access_token,
internal_emails=internal_emails,
)
payload = _response_json(response, "Gmail history")
message_ids = list(
dict.fromkeys(
str(added["message"]["id"])
for history in payload.get("history", [])
for added in history.get("messagesAdded", [])
if added.get("message", {}).get("id")
)
)
_save_history_progress(
integration,
start_history_id=progress_start_id,
page_token=str(payload.get("nextPageToken") or "") or None,
target_history_id=str(payload.get("historyId") or target_history_id or progress_start_id),
pending_message_ids=message_ids,
)
return None
def _save_history_progress(
integration: IntegrationRecord,
*,
start_history_id: str,
page_token: str | None,
target_history_id: str | None,
pending_message_ids: list[str],
) -> None:
IntegrationManager.google_calendar.oauth.update(
integration.team_id,
integration.id,
config_patch={
GMAIL_HISTORY_START_ID_CONFIG_KEY: start_history_id,
GMAIL_HISTORY_PAGE_TOKEN_CONFIG_KEY: page_token,
GMAIL_HISTORY_TARGET_ID_CONFIG_KEY: target_history_id,
GMAIL_PENDING_MESSAGE_IDS_CONFIG_KEY: pending_message_ids,
},
)
def _clear_history_progress(integration: IntegrationRecord) -> None:
IntegrationManager.google_calendar.oauth.update(
integration.team_id,
integration.id,
config_remove=(
GMAIL_HISTORY_START_ID_CONFIG_KEY,
GMAIL_HISTORY_PAGE_TOKEN_CONFIG_KEY,
GMAIL_HISTORY_TARGET_ID_CONFIG_KEY,
GMAIL_PENDING_MESSAGE_IDS_CONFIG_KEY,
),
)
def _ingest_message_ids(
*,
integration: IntegrationRecord,
channel: EmailChannel,
access_token: str,
message_ids: list[str],
internal_emails: set[str],
) -> None:
for gmail_message_id in message_ids:
_ingest_message_id(
integration=integration,
channel=channel,
access_token=access_token,
gmail_message_id=gmail_message_id,
internal_emails=internal_emails,
)
def _ingest_message_id(
*,
integration: IntegrationRecord,
channel: EmailChannel,
access_token: str,
gmail_message_id: str,
internal_emails: set[str],
) -> None:
payload = _get_json_allowing_missing(
integration=integration,
access_token=access_token,
url=f"{GMAIL_API_BASE_URL}/messages/{gmail_message_id}",
endpoint="/gmail/v1/users/me/messages/{message_id}",
params={"format": "full"},
)
# The user deleted the message between the history page and this fetch. Skip it.
if payload is None:
return
def load_attachment_data(attachment_id: str) -> str:
attachment = _get_json_allowing_missing(
integration=integration,
access_token=access_token,
url=f"{GMAIL_API_BASE_URL}/messages/{gmail_message_id}/attachments/{attachment_id}",
endpoint="/gmail/v1/users/me/messages/{message_id}/attachments/{attachment_id}",
)
return str(attachment.get("data") or "") if attachment else ""
try:
parsed = _parse_gmail_message(
payload,
channel.from_email,
str(integration.integration_id),
load_attachment_data=load_attachment_data,
)
except _GmailBodyPartLimitExceeded:
return
if parsed is None or not _has_external_participant(parsed, internal_emails):
return
labels = set(payload.get("labelIds") or [])
if not labels.intersection({"INBOX", "SENT"}):
return
direction = EmailThreadMessageDirection.OUTBOUND if "SENT" in labels else EmailThreadMessageDirection.INBOUND
ingest_customer_email(
team_id=integration.team_id,
channel=channel,
email=parsed,
direction=direction,
source_type="gmail",
source_id=f"{integration.id}:{gmail_message_id}",
)
def _parse_gmail_message(
payload: dict[str, Any],
mailbox_email: str,
google_account_id: str,
*,
load_attachment_data: Callable[[str], str] | None = None,
) -> ParsedEmail | None:
gmail_message_id = str(payload.get("id") or "")
message_payload = payload.get("payload") or {}
headers = _message_headers(message_payload.get("headers") or [])
sender_name, sender_email = parseaddr(headers.get("from", ""))
sender_email = sender_email.strip().lower()[:400]
if not gmail_message_id or not sender_email:
return None
message_id = headers.get("message-id") or f"gmail:{google_account_id}:{gmail_message_id}"
in_reply_to_ids = _parse_message_ids(headers.get("in-reply-to", ""))
sent_at = _parse_internal_date(payload.get("internalDate"))
bodies = _message_bodies(message_payload, load_attachment_data=load_attachment_data)
body_plain = bodies.plain
if not body_plain and bodies.html:
body_plain = _html_to_text(bodies.html)
return ParsedEmail(
message_id=message_id[:998],
in_reply_to=in_reply_to_ids[0] if in_reply_to_ids else None,
references=_parse_message_ids(headers.get("references", "")),
sent_at=sent_at,
sender=EmailAddress(name=_decode_header(sender_name)[:400], email=sender_email),
to_recipients=_parse_addresses(headers.get("to", "")),
cc_recipients=_parse_addresses(headers.get("cc", "")),
subject=_decode_header(headers.get("subject", ""))[:500],
body_plain=body_plain[:50_000],
stripped_text=body_plain[:50_000],
body_html=(bodies.html or "")[:50_000],
sender_authenticated=False,
dkim_passed=False,
dkim_signing_domains=(),
capture_address=mailbox_email,
attachments=(),
)
def _html_to_text(html: str) -> str:
"""Flatten an HTML-only body to plain text, keeping paragraph and line breaks.
`strip_tags` joins adjacent blocks with no separator, so `<p>a.</p><p>b</p>`
becomes `a.b`. Links are folded back in later by `recover_links_from_html`.
"""
preformatted: list[str] = []
try:
text = _flatten_html(BeautifulSoup(html, "html.parser"), preformatted)
except Exception: # noqa: BLE001 — a malformed HTML part must never fail the sync
preformatted = []
text = unescape(strip_tags(html))
lines = (line.strip() for line in text.split("\n"))
text = _BLANK_LINES_RE.sub("\n\n", "\n".join(lines)).strip()
# Preformatted blocks go back in after the cleanup, so their indentation survives.
return _PREFORMATTED_PLACEHOLDER_RE.sub(lambda match: _get_preformatted_block(preformatted, match), text)
@frozen
class _HtmlSeparator:
text: str
def _flatten_html(soup: BeautifulSoup, preformatted: list[str]) -> str:
"""Emit the text of `soup` in one pass, replacing each top-level `<pre>` with a placeholder.
The walk reads the tree without modifying it, because BeautifulSoup locates a
node among its siblings with a linear scan on every insert or replace, which
makes per-node edits quadratic on an email with many sibling elements.
"""
parts: list[str] = []
stack: list[PageElement | _HtmlSeparator] = list(reversed(soup.contents))
while stack:
node = stack.pop()
if isinstance(node, _HtmlSeparator):
parts.append(node.text)
elif isinstance(node, NavigableString):
if type(node) is NavigableString:
parts.append(_HTML_WHITESPACE_RE.sub(" ", node).replace("\0", ""))
elif isinstance(node, Tag) and node.name not in _HTML_SKIPPED_TAGS:
if node.name == "br":
parts.append("\n")
elif node.name == "pre":
preformatted.append(_preformatted_text(node))
parts.append(f"\0{len(preformatted) - 1}\0\n\n")
else:
if node.name in _HTML_PARAGRAPH_TAGS:
stack.append(_HtmlSeparator(text="\n\n"))
elif node.name in _HTML_LINE_TAGS:
stack.append(_HtmlSeparator(text="\n"))
stack.extend(reversed(node.contents))
return "".join(parts)
def _preformatted_text(pre: Tag) -> str:
parts: list[str] = []
for node in pre.descendants:
if type(node) is NavigableString:
parts.append(node.replace("\0", ""))
elif isinstance(node, Tag) and node.name == "br":
parts.append("\n")
return "".join(parts).strip("\n")
def _get_preformatted_block(preformatted: list[str], match: re.Match[str]) -> str:
index = int(match.group(1))
return preformatted[index] if index < len(preformatted) else ""
def _message_headers(raw_headers: list[dict[str, Any]]) -> dict[str, str]:
values: dict[str, list[str]] = {}
for header in raw_headers:
name = str(header.get("name") or "").lower()
value = str(header.get("value") or "")
if name and value:
values.setdefault(name, []).append(value)
return {name: ", ".join(items) for name, items in values.items()}
def _decode_header(value: str) -> str:
try:
return str(make_header(decode_header(value)))
except (LookupError, UnicodeError):
return value
def _parse_message_ids(value: str) -> tuple[str, ...]:
message_ids = _MESSAGE_ID_RE.findall(value)
if not message_ids:
message_ids = value.split()
return tuple(dict.fromkeys(message_id.strip()[:998] for message_id in message_ids if message_id.strip()))
def _parse_addresses(value: str) -> tuple[EmailAddress, ...]:
addresses: list[EmailAddress] = []
seen: set[str] = set()
for name, email in getaddresses([value]):
normalized = email.strip().lower()[:400]
if not normalized or normalized in seen:
continue
seen.add(normalized)
addresses.append(EmailAddress(name=_decode_header(name)[:400], email=normalized))
return tuple(addresses)
def _parse_internal_date(value: Any) -> datetime:
try:
return datetime.fromtimestamp(int(str(value)) / 1000, tz=UTC)
except (TypeError, ValueError, OverflowError):
return timezone.now()
def _message_bodies(
payload: dict[str, Any],
*,
load_attachment_data: Callable[[str], str] | None = None,
) -> _EmailBodies:
plain_parts: list[str] = []
html_parts: list[str] = []
attachment_backed_body_parts = 0
def visit(part: dict[str, Any]) -> None:
nonlocal attachment_backed_body_parts
mime_type = str(part.get("mimeType") or "").lower()
body = part.get("body") or {}
data = str(body.get("data") or "")
attachment_id = str(body.get("attachmentId") or "")
if mime_type in ("text/plain", "text/html") and not data and attachment_id and load_attachment_data:
if attachment_backed_body_parts >= MAX_ATTACHMENT_BACKED_BODY_PARTS:
raise _GmailBodyPartLimitExceeded
attachment_backed_body_parts += 1
data = load_attachment_data(attachment_id)
decoded = _decode_body(data)
if decoded and mime_type == "text/plain":
plain_parts.append(decoded)
elif decoded and mime_type == "text/html":
html_parts.append(decoded)
for child in part.get("parts") or []:
visit(child)
visit(payload)
return _EmailBodies(
plain="\n".join(plain_parts).strip(),
html="\n".join(html_parts).strip(),
)
def _decode_body(data: str) -> str:
if not data:
return ""
try:
padded = data + "=" * (-len(data) % 4)
return base64.urlsafe_b64decode(padded).decode("utf-8", errors="replace")
except (ValueError, binascii.Error):
return ""
def _has_external_participant(email: ParsedEmail, internal_emails: set[str]) -> bool:
addresses = {
email.sender.email,
*(recipient.email for recipient in email.to_recipients),
*(recipient.email for recipient in email.cc_recipients),
}
return any(address and address not in internal_emails for address in addresses)
def _request(
*,
integration: IntegrationRecord,
access_token: str,
url: str,
endpoint: str,
params: dict[str, Any] | None = None,
) -> requests.Response:
return google_workspace_request(
"GET",
url,
access_token=access_token,
account_id=str(integration.integration_id),
source="customer_analytics",
endpoint=endpoint,
params=params,
timeout=30,
)
def _get_json(
*,
integration: IntegrationRecord,
access_token: str,
url: str,
endpoint: str,
params: dict[str, Any] | None = None,
) -> dict[str, Any]:
response = _request(
integration=integration,
access_token=access_token,
url=url,
endpoint=endpoint,
params=params,
)
return _response_json(response, "Gmail API")
def _get_json_allowing_missing(
*,
integration: IntegrationRecord,
access_token: str,
url: str,
endpoint: str,
params: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
response = _request(
integration=integration,
access_token=access_token,
url=url,
endpoint=endpoint,
params=params,
)
if response.status_code == 404:
return None
return _response_json(response, "Gmail API")
def _response_json(response: requests.Response, operation: str) -> dict[str, Any]:
if response.status_code != 200:
raise GmailSyncError(f"{operation} returned {response.status_code}: {response.text[:200]}")
try:
payload = response.json()
except ValueError as error:
raise GmailSyncError(f"{operation} returned an invalid response") from error
if not isinstance(payload, dict):
raise GmailSyncError(f"{operation} returned an invalid response")
return payload