Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the SharePoint source (#123718)

Tom Owersff0140af9eaa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index d19968805d15..811202d42fc9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10895,10 +10895,10 @@ Today (5): `drive_items`, `drives`, `list_items`, `lists`, `sites`
Diffed against: <https://learn.microsoft.com/en-us/graph/api/resources/driveitem?view=graph-rest-1.0>
-- [ ] `Site content types (GET /v1.0/sites/{site-id}/contentTypes)` — resolve list-item contentType IDs to content definitions for consistent classification across lists (high)
-- [ ] `Activity statistics by interval (GET /v1.0/sites/{site-id}/getActivitiesByInterval(startDateTime={startDateTime},endDateTime={endDateTime},interval={interval}))` — measure site engagement and view activity over time, which metadata tables cannot provide (high)
-- [ ] `Drive item versions (GET /v1.0/drives/{drive-id}/items/{item-id}/versions)` — track document revisions, editors, and historical file sizes (medium)
-- [ ] `List item versions (GET /v1.0/sites/{site-id}/lists/{list-id}/items/{item-id}/versions)` — analyze changes to business records and the people who edited them (medium)
+- [x] `Site content types (GET /v1.0/sites/{site-id}/contentTypes)` — resolve list-item contentType IDs to content definitions for consistent classification across lists (high)
+- [x] `Activity statistics by interval (GET /v1.0/sites/{site-id}/getActivitiesByInterval(startDateTime={startDateTime},endDateTime={endDateTime},interval={interval}))` — measure site engagement and view activity over time, which metadata tables cannot provide (high)
+- [x] `Drive item versions (GET /v1.0/drives/{drive-id}/items/{item-id}/versions)` — track document revisions, editors, and historical file sizes (medium)
+- [x] `List item versions (GET /v1.0/sites/{site-id}/lists/{list-id}/items/{item-id}/versions)` — analyze changes to business records and the people who edited them (medium)
- [ ] `Recent item activities (GET /v1.0/drives/{drive-id}/items/{item-id}/activities)` — analyze recent user actions on documents instead of only their latest modification timestamps (medium)
- [ ] `Drive item permissions (GET /v1.0/drives/{drive-id}/items/{item-id}/permissions)` — analyze document sharing, granted access, and links across document libraries (medium)
- [ ] `Site pages (GET /v1.0/sites/{site-id}/pages/microsoft.graph.sitePage)` — analyze published pages, authorship, and publishing state separately from file metadata (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/canonical_descriptions.py
index 20c39c6782f1..6ca019923494 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/canonical_descriptions.py
@@ -104,4 +104,71 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"root": "Present when the item is the root folder of the drive.",
},
},
+ "content_types": {
+ "description": "A content type defined in a SharePoint site: a reusable set of columns and settings for list items and documents.",
+ "docs_url": "https://learn.microsoft.com/en-us/graph/api/resources/contenttype",
+ "columns": {
+ "site_id": "Identifier of the site the content type belongs to.",
+ "id": "Identifier of the content type. List items reference it in contentType.",
+ "name": "Name of the content type.",
+ "description": "Description of the content type.",
+ "group": "Group the content type belongs to, used to organize content types.",
+ "hidden": "Whether the content type is hidden in the list's New menu.",
+ "readOnly": "Whether the content type can be modified.",
+ "sealed": "Whether the content type can be modified by users or through push-down operations.",
+ "isBuiltIn": "Whether the content type is one of SharePoint's built-in content types.",
+ "parentId": "Identifier of the content type this one inherits from.",
+ "base": "Parent content type this one is derived from.",
+ "inheritedFrom": "Reference to the site or list where the content type is defined, when it is inherited.",
+ "propagateChanges": "Whether changes to the content type are pushed to inherited content types and lists that use it.",
+ "associatedHubsUrls": "URLs of the hub sites the content type is associated with.",
+ "documentSet": "Document set settings, when the content type is a document set.",
+ "documentTemplate": "Document template of the content type.",
+ "order": "Order of the content type in the list's New menu.",
+ },
+ },
+ "site_activity_stats": {
+ "description": "Activity counts for a SharePoint site over one day.",
+ "docs_url": "https://learn.microsoft.com/en-us/graph/api/resources/itemactivitystat",
+ "columns": {
+ "site_id": "Identifier of the site the activity took place in.",
+ "startDateTime": "Start of the day the counts cover.",
+ "endDateTime": "End of the day the counts cover.",
+ "access": "Number of access actions and of distinct users who did them.",
+ "create": "Number of create actions and of distinct users who did them.",
+ "edit": "Number of edit actions and of distinct users who did them.",
+ "delete": "Number of delete actions and of distinct users who did them.",
+ "move": "Number of move actions and of distinct users who did them.",
+ "isTrending": "Whether the site is trending.",
+ "incompleteData": "Present when the counts for the interval may be incomplete.",
+ },
+ },
+ "drive_item_versions": {
+ "description": "A previous or current version of a file in a SharePoint document library.",
+ "docs_url": "https://learn.microsoft.com/en-us/graph/api/resources/driveitemversion",
+ "columns": {
+ "site_id": "Identifier of the site the drive belongs to.",
+ "drive_id": "Identifier of the drive (document library) the file belongs to.",
+ "item_id": "Identifier of the file the version belongs to.",
+ "id": "Identifier of the version, such as 3.0, unique within its file.",
+ "lastModifiedBy": "Identity of the user who last modified the version.",
+ "lastModifiedDateTime": "Date and time the version was last modified.",
+ "size": "Size of the version in bytes.",
+ "publication": "Publication status of the version.",
+ },
+ },
+ "list_item_versions": {
+ "description": "A previous or current version of an item in a SharePoint list.",
+ "docs_url": "https://learn.microsoft.com/en-us/graph/api/resources/listitemversion",
+ "columns": {
+ "site_id": "Identifier of the site the item's list belongs to.",
+ "list_id": "Identifier of the list the item belongs to.",
+ "item_id": "Identifier of the list item the version belongs to.",
+ "id": "Identifier of the version, such as 3.0, unique within its item.",
+ "lastModifiedBy": "Identity of the user who last modified the version.",
+ "lastModifiedDateTime": "Date and time the version was last modified.",
+ "published": "Publication status of the version.",
+ "fields": "Values of the list's columns in this version, keyed by column internal name.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/settings.py
index 81cc6dc76a16..f400da390add 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/settings.py
@@ -1,8 +1,9 @@
from dataclasses import dataclass
+from datetime import timedelta
from enum import StrEnum
from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
GRAPH_BASE_URL = "https://graph.microsoft.com/v1.0"
# Entra ID (Azure AD) token endpoint; the tenant id is interpolated into the path.
@@ -25,6 +26,12 @@ PATTERN_ERROR = "The file pattern isn't a valid regular expression"
SITE_ID_COLUMN = "site_id"
LIST_ID_COLUMN = "list_id"
DRIVE_ID_COLUMN = "drive_id"
+ITEM_ID_COLUMN = "item_id"
+
+# Graph rejects a daily activity range of 90 days or more.
+ACTIVITY_STATS_WINDOW = timedelta(days=89)
+# Graph aggregates activity late, so each incremental sync reads the last days again.
+ACTIVITY_STATS_INCREMENTAL_LOOKBACK_SECONDS = 2 * 24 * 60 * 60
class SharePointEndpoint(StrEnum):
@@ -33,6 +40,10 @@ class SharePointEndpoint(StrEnum):
LIST_ITEMS = "list_items"
DRIVES = "drives"
DRIVE_ITEMS = "drive_items"
+ CONTENT_TYPES = "content_types"
+ SITE_ACTIVITY_STATS = "site_activity_stats"
+ DRIVE_ITEM_VERSIONS = "drive_item_versions"
+ LIST_ITEM_VERSIONS = "list_item_versions"
@dataclass(frozen=True)
@@ -42,6 +53,7 @@ class SharePointEndpointConfig:
description: str
# Stable datetime column for datetime partitioning (None = no datetime partitioning).
partition_key: Optional[str] = None
+ should_sync_default: bool = True
SHAREPOINT_ENDPOINTS: dict[str, SharePointEndpointConfig] = {
@@ -77,10 +89,55 @@ SHAREPOINT_ENDPOINTS: dict[str, SharePointEndpointConfig] = {
description="File and folder metadata in every document library",
partition_key="createdDateTime",
),
+ SharePointEndpoint.CONTENT_TYPES: SharePointEndpointConfig(
+ name=SharePointEndpoint.CONTENT_TYPES,
+ # Built-in content type ids such as `0x0101` repeat in every site.
+ primary_keys=[SITE_ID_COLUMN, "id"],
+ description="Content types defined in each site, to resolve the content type of list items",
+ ),
+ SharePointEndpoint.SITE_ACTIVITY_STATS: SharePointEndpointConfig(
+ name=SharePointEndpoint.SITE_ACTIVITY_STATS,
+ primary_keys=[SITE_ID_COLUMN, "startDateTime"],
+ description=(
+ "Daily access, create, edit, delete, and move counts for each site. Graph keeps the last 89 days, "
+ "so sync incrementally to keep older days"
+ ),
+ partition_key="startDateTime",
+ ),
+ SharePointEndpoint.DRIVE_ITEM_VERSIONS: SharePointEndpointConfig(
+ name=SharePointEndpoint.DRIVE_ITEM_VERSIONS,
+ # Version ids such as `1.0` repeat for every file.
+ primary_keys=[DRIVE_ID_COLUMN, ITEM_ID_COLUMN, "id"],
+ # No partition key: the current version keeps its id while it is edited, so its modified
+ # time changes.
+ description=("Version history of every file in every document library. Each sync makes one request per file"),
+ # One request per file or list item is too costly to turn on without the user choosing it.
+ should_sync_default=False,
+ ),
+ SharePointEndpoint.LIST_ITEM_VERSIONS: SharePointEndpointConfig(
+ name=SharePointEndpoint.LIST_ITEM_VERSIONS,
+ primary_keys=[SITE_ID_COLUMN, LIST_ID_COLUMN, ITEM_ID_COLUMN, "id"],
+ description=(
+ "Version history of every item in every visible list, with the column values of each version in "
+ "`fields`. Each sync makes one request per item"
+ ),
+ should_sync_default=False,
+ ),
}
ENDPOINTS = tuple(SHAREPOINT_ENDPOINTS.keys())
# Graph has no reliable server-side "modified since" filter across lists and drives (list
-# filters only work on indexed columns), so every table is full refresh.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: [] for name in ENDPOINTS}
+# filters only work on indexed columns), so those tables are full refresh. Activity stats take
+# the start of their date range in the request.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ **{name: [] for name in ENDPOINTS},
+ SharePointEndpoint.SITE_ACTIVITY_STATS: [
+ IncrementalField(
+ label="startDateTime",
+ type=IncrementalFieldType.DateTime,
+ field="startDateTime",
+ field_type=IncrementalFieldType.DateTime,
+ )
+ ],
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/sharepoint.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/sharepoint.py
index 0373e8d3809f..220e794592d4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/sharepoint.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/sharepoint.py
@@ -1,6 +1,6 @@
import dataclasses
from collections.abc import Callable, Iterator
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from typing import Any, Optional
from urllib.parse import quote, urlparse
@@ -13,8 +13,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.sharepoint.settings import (
+ ACTIVITY_STATS_WINDOW,
DRIVE_ID_COLUMN,
GRAPH_BASE_URL,
+ INCREMENTAL_FIELDS,
+ ITEM_ID_COLUMN,
LIST_ID_COLUMN,
LOGIN_BASE_URL,
MAX_PAGES_PER_COLLECTION,
@@ -29,6 +32,9 @@ VALIDATE_TIMEOUT_SECONDS = 30
_module_logger: FilteringBoundLogger = structlog.get_logger(__name__)
+LIST_ENDPOINTS = (SharePointEndpoint.LIST_ITEMS, SharePointEndpoint.LIST_ITEM_VERSIONS)
+VERSION_ENDPOINTS = (SharePointEndpoint.DRIVE_ITEM_VERSIONS, SharePointEndpoint.LIST_ITEM_VERSIONS)
+
SITES_DENIED_ERROR = (
"Microsoft Graph denied access to SharePoint. Grant the app the Sites.Read.All application permission "
"with admin consent, or list the site URLs it was granted through Sites.Selected."
@@ -390,7 +396,13 @@ def _iter_grandchildren(
def stage(next_url: str, site_id: str = site_id, parent_id: str = parent_id) -> None:
manager.save_state(SharePointResumeConfig(site_id=site_id, parent_id=parent_id, next_url=next_url))
- for page in _iter_parent_rows(client, endpoint, quoted_site, site_id, parent_id, start_url, stage, logger):
+ if endpoint in VERSION_ENDPOINTS:
+ pages = _iter_parent_versions(
+ client, endpoint, quoted_site, site_id, parent_id, start_url, stage, manager.safe_point, logger
+ )
+ else:
+ pages = _iter_parent_rows(client, endpoint, quoted_site, site_id, parent_id, start_url, stage, logger)
+ for page in pages:
yielded = True
yield page
start_url = None
@@ -401,13 +413,14 @@ def _iter_grandchildren(
def _site_parents(
client: SharePointClient, endpoint: SharePointEndpoint, quoted_site: str, logger: FilteringBoundLogger
) -> list[dict[str, Any]]:
- collection = "lists" if endpoint == SharePointEndpoint.LIST_ITEMS else "drives"
+ is_list = endpoint in LIST_ENDPOINTS
+ collection = "lists" if is_list else "drives"
parents: list[dict[str, Any]] = []
path = f"/sites/{quoted_site}/{collection}"
for page in _walk_child_collection(client, path, None, None, lambda _: None, logger):
for parent in page:
# Hidden lists are SharePoint's own plumbing (workflow history, user info, and so on).
- if endpoint == SharePointEndpoint.LIST_ITEMS and (parent.get("list") or {}).get("hidden"):
+ if is_list and (parent.get("list") or {}).get("hidden"):
continue
parents.append(parent)
return parents
@@ -451,6 +464,85 @@ def _iter_parent_rows(
yield rows
+def _iter_parent_versions(
+ client: SharePointClient,
+ endpoint: SharePointEndpoint,
+ quoted_site: str,
+ site_id: str,
+ parent_id: str,
+ start_url: Optional[str],
+ stage: Callable[[str], None],
+ safe_point: Callable[[], None],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ """Versions of every item in one list (`list_item_versions`) or every file in one drive (`drive_item_versions`).
+
+ The cursor names the page of items in progress, not the next one, because the versions of
+ that page are still being read. A resume reads the versions of that page again.
+ """
+ quoted_parent = quote(parent_id, safe="!-_")
+ is_list = endpoint == SharePointEndpoint.LIST_ITEM_VERSIONS
+ if is_list:
+ items_path = f"/sites/{quoted_site}/lists/{quoted_parent}/items"
+ item_prefix = items_path
+ version_params: Optional[dict[str, str]] = {"expand": "fields"}
+ parent_columns = {SITE_ID_COLUMN: site_id, LIST_ID_COLUMN: parent_id}
+ else:
+ items_path = f"/drives/{quoted_parent}/root/delta"
+ item_prefix = f"/drives/{quoted_parent}/items"
+ version_params = None
+ parent_columns = {SITE_ID_COLUMN: site_id, DRIVE_ID_COLUMN: parent_id}
+
+ next_url: Optional[str] = None
+
+ def remember_next(url: str) -> None:
+ nonlocal next_url
+ next_url = url
+
+ page_url = start_url
+ seen: set[str] = set()
+ for page in _walk_child_collection(client, items_path, None, start_url, remember_next, logger):
+ if page_url is not None:
+ stage(page_url)
+ page_yielded = False
+ for item in page:
+ item_id = item.get("id")
+ if item_id is None or item_id in seen:
+ continue
+ # Folders have no versions, and delta can return an item more than once.
+ if not is_list and (item.get("deleted") is not None or item.get("file") is None):
+ continue
+ seen.add(item_id)
+ path = f"{item_prefix}/{quote(item_id, safe='!-_')}/versions"
+ item_columns = {**parent_columns, ITEM_ID_COLUMN: item_id}
+ item_yielded = False
+ for versions in _walk_child_collection(client, path, version_params, None, lambda _: None, logger):
+ rows = []
+ for version in versions:
+ version.pop("@microsoft.graph.downloadUrl", None)
+ rows.append(_normalize(version, endpoint, **item_columns))
+ item_yielded = True
+ yield rows
+ if item_yielded:
+ page_yielded = True
+ else:
+ safe_point()
+ if not page_yielded:
+ safe_point()
+ page_url = next_url
+
+
+def _activity_stats_collection(last_value: Any, now: datetime) -> str:
+ end = (now.astimezone(UTC) + timedelta(days=1)).date()
+ start = end - ACTIVITY_STATS_WINDOW
+ watermark = _parse_datetime(last_value)
+ if watermark is not None:
+ start = max(start, watermark.astimezone(UTC).date())
+ return (
+ f"getActivitiesByInterval(startDateTime='{start.isoformat()}',endDateTime='{end.isoformat()}',interval='day')"
+ )
+
+
def _get_rows(
tenant_id: str,
client_id: str,
@@ -459,6 +551,7 @@ def _get_rows(
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[SharePointResumeConfig],
+ incremental_field_last_value: Any = None,
) -> Iterator[list[dict[str, Any]]]:
sharepoint_endpoint = SharePointEndpoint(endpoint)
site_paths = parse_site_urls(site_urls)
@@ -471,9 +564,26 @@ def _get_rows(
sites = _list_sites(client, site_paths, logger)
- if sharepoint_endpoint in (SharePointEndpoint.LISTS, SharePointEndpoint.DRIVES):
+ site_collections = {
+ SharePointEndpoint.LISTS: "lists",
+ SharePointEndpoint.DRIVES: "drives",
+ SharePointEndpoint.CONTENT_TYPES: "contentTypes",
+ }
+ if sharepoint_endpoint == SharePointEndpoint.SITE_ACTIVITY_STATS:
+ collection = _activity_stats_collection(incremental_field_last_value, datetime.now(UTC))
+ yield from _iter_site_children(
+ client, sharepoint_endpoint, collection, sites, logger, resumable_source_manager, resume
+ )
+ return
+ if sharepoint_endpoint in site_collections:
yield from _iter_site_children(
- client, sharepoint_endpoint, sharepoint_endpoint.value, sites, logger, resumable_source_manager, resume
+ client,
+ sharepoint_endpoint,
+ site_collections[sharepoint_endpoint],
+ sites,
+ logger,
+ resumable_source_manager,
+ resume,
)
return
@@ -488,6 +598,7 @@ def sharepoint_source(
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[SharePointResumeConfig],
+ incremental_field_last_value: Any = None,
) -> SourceResponse:
config = SHAREPOINT_ENDPOINTS[endpoint]
has_datetime_partition = config.partition_key is not None
@@ -502,6 +613,7 @@ def sharepoint_source(
endpoint=endpoint,
logger=logger,
resumable_source_manager=resumable_source_manager,
+ incremental_field_last_value=incremental_field_last_value,
),
primary_keys=config.primary_keys,
partition_count=1,
@@ -509,7 +621,8 @@ def sharepoint_source(
partition_mode="datetime" if has_datetime_partition else None,
partition_format="month" if has_datetime_partition else None,
partition_keys=[config.partition_key] if config.partition_key is not None else None,
- sort_mode="asc",
+ # Incremental activity stats are ascending per site, not across the whole run.
+ sort_mode=None if INCREMENTAL_FIELDS[endpoint] else "asc",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/source.py
index 41cb18c5a3ad..abadd4159f30 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/source.py
@@ -35,11 +35,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.sharepoint
sharepoint_file_source,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.sharepoint.settings import (
+ ACTIVITY_STATS_INCREMENTAL_LOOKBACK_SECONDS,
ENDPOINTS,
FILE_NOT_FOUND_ERROR,
INCREMENTAL_FIELDS,
PATTERN_ERROR,
SHAREPOINT_ENDPOINTS,
+ SharePointEndpoint,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.sharepoint.sharepoint import (
INVALID_SITE_URL_ERROR,
@@ -122,7 +124,13 @@ class SharePointSource(ResumableSource[SharePointSourceConfig, SharePointResumeC
schemas = build_endpoint_schemas(
ENDPOINTS,
INCREMENTAL_FIELDS,
+ # Each sync reads the latest days again, so append would duplicate them.
+ merge_only=(SharePointEndpoint.SITE_ACTIVITY_STATS,),
descriptions={name: endpoint.description for name, endpoint in SHAREPOINT_ENDPOINTS.items()},
+ should_sync_default={name: endpoint.should_sync_default for name, endpoint in SHAREPOINT_ENDPOINTS.items()},
+ default_incremental_lookback_seconds={
+ SharePointEndpoint.SITE_ACTIVITY_STATS: ACTIVITY_STATS_INCREMENTAL_LOOKBACK_SECONDS
+ },
)
import_files = config.import_files
if (
@@ -198,6 +206,9 @@ class SharePointSource(ResumableSource[SharePointSourceConfig, SharePointResumeC
endpoint=inputs.schema_name,
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
+ incremental_field_last_value=(
+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None
+ ),
)
@property
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/tests/test_sharepoint.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/tests/test_sharepoint.py
index 54dd937dfebe..a9d5f1606401 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/tests/test_sharepoint.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/tests/test_sharepoint.py
@@ -8,6 +8,7 @@ from http import HTTPStatus
from typing import Any, Optional, Union, cast
import pytest
+import time_machine
from unittest import mock
import requests
@@ -163,9 +164,10 @@ def _run(
routes: Mapping[str, Route | list[Route]],
site_urls: Optional[str] = None,
resume_state: Optional[SharePointResumeConfig] = None,
+ **inputs: Any,
) -> DriveResult:
driver = SourceDriver(SharePointSource(), make_config(enabled=None, site_urls=site_urls))
- return driver.run(endpoint, _graph(routes), resume_state=resume_state)
+ return driver.run(endpoint, _graph(routes), resume_state=resume_state, **inputs)
ALL_SITES_ROUTES: dict[str, Route] = {
@@ -335,6 +337,183 @@ class TestSharePointResume:
assert [(row["site_id"], row["id"]) for row in result.rows] == [(SITE_A, "list-a"), (SITE_B, "list-b")]
+SITE_A_URL = "https://contoso.sharepoint.com/sites/a"
+SITE_A_ROUTE: dict[str, Route] = {_g("/sites/contoso.sharepoint.com:/sites/a:"): {"id": SITE_A}}
+
+
+def _activity_path(start: str) -> str:
+ return _g(
+ f"/sites/{SITE_A}/getActivitiesByInterval(startDateTime='{start}',endDateTime='2026-10-27',interval='day')"
+ )
+
+
+class TestSharePointSiteTables:
+ @parameterized.expand(
+ [
+ ("drives", "drives"),
+ ("content_types", "contentTypes"),
+ ]
+ )
+ def test_reads_each_site_collection(self, endpoint: str, collection: str) -> None:
+ routes: dict[str, Route] = {
+ **SITE_A_ROUTE,
+ _g(f"/sites/{SITE_A}/{collection}"): {
+ "value": [{"id": "0x0101", "name": "Document"}],
+ "@odata.nextLink": _g(f"/sites/{SITE_A}/{collection}?$skiptoken=2"),
+ },
+ _g(f"/sites/{SITE_A}/{collection}?$skiptoken=2"): {"value": [{"id": "0x0120", "name": "Folder"}]},
+ }
+
+ result = _run(endpoint, routes, site_urls=SITE_A_URL)
+
+ assert result.raised is None
+ assert [(row["site_id"], row["id"]) for row in result.rows] == [(SITE_A, "0x0101"), (SITE_A, "0x0120")]
+
+ @parameterized.expand(
+ [
+ ("full_refresh_reads_the_whole_window", {}, "2026-07-30"),
+ (
+ "incremental_starts_at_the_watermark_day",
+ {"incremental_field": "startDateTime", "db_incremental_field_last_value": "2026-10-20T00:00:00Z"},
+ "2026-10-20",
+ ),
+ (
+ "incremental_watermark_as_datetime",
+ {
+ "incremental_field": "startDateTime",
+ "db_incremental_field_last_value": datetime(2026, 10, 25, 6, tzinfo=UTC),
+ },
+ "2026-10-25",
+ ),
+ (
+ "incremental_watermark_older_than_the_window_is_clamped",
+ {"incremental_field": "startDateTime", "db_incremental_field_last_value": "2026-01-01T00:00:00Z"},
+ "2026-07-30",
+ ),
+ ]
+ )
+ def test_activity_stats_request_a_daily_window_graph_accepts(
+ self, _name: str, inputs: dict[str, Any], expected_start: str
+ ) -> None:
+ routes: dict[str, Route] = {
+ **SITE_A_ROUTE,
+ _activity_path(expected_start): {
+ "value": [
+ {
+ "startDateTime": "2026-10-25T00:00:00Z",
+ "endDateTime": "2026-10-26T00:00:00Z",
+ "access": {"actionCount": 5, "actorCount": 3},
+ }
+ ]
+ },
+ }
+
+ with time_machine.travel(datetime(2026, 10, 26, 23, 30, tzinfo=UTC), tick=False):
+ result = _run("site_activity_stats", routes, site_urls=SITE_A_URL, **inputs)
+
+ assert result.raised is None
+ assert result.rows == [
+ {
+ "site_id": SITE_A,
+ "startDateTime": datetime(2026, 10, 25, tzinfo=UTC),
+ "endDateTime": "2026-10-26T00:00:00Z",
+ "access": {"actionCount": 5, "actorCount": 3},
+ }
+ ]
+ assert result.response is not None and result.response.sort_mode is None
+
+
+DRIVE_VERSION_ROUTES: dict[str, Route] = {
+ **SITE_A_ROUTE,
+ _g(f"/sites/{SITE_A}/drives"): {"value": [{"id": "b!drive-a"}]},
+ _g("/drives/b!drive-a/root/delta"): {
+ "value": [
+ {"id": "root", "root": {}, "folder": {}},
+ {"id": "f1", "file": {}},
+ {"id": "gone", "deleted": {}},
+ ],
+ "@odata.nextLink": _g("/drives/b!drive-a/root/delta?token=2"),
+ },
+ _g("/drives/b!drive-a/root/delta?token=2"): {
+ "value": [{"id": "f1", "file": {}}, {"id": "f2", "file": {}}, {"id": "f3", "file": {}}],
+ "@odata.nextLink": _g("/drives/b!drive-a/root/delta?token=3"),
+ },
+ _g("/drives/b!drive-a/root/delta?token=3"): {"value": [{"id": "folder-2", "folder": {}}]},
+ _g("/drives/b!drive-a/items/f1/versions"): {
+ "value": [{"id": "2.0", "size": 20, "@microsoft.graph.downloadUrl": "https://temporary.example.com/f1"}],
+ "@odata.nextLink": _g("/drives/b!drive-a/items/f1/versions?$skiptoken=2"),
+ },
+ _g("/drives/b!drive-a/items/f1/versions?$skiptoken=2"): {"value": [{"id": "1.0", "size": 10}]},
+ # A file deleted after the delta page was read.
+ _g("/drives/b!drive-a/items/f2/versions"): 404,
+ _g("/drives/b!drive-a/items/f3/versions"): {"value": [{"id": "1.0", "size": 30}]},
+}
+
+
+class TestSharePointVersions:
+ def test_drive_item_versions_read_each_file_once(self) -> None:
+ result = _run("drive_item_versions", DRIVE_VERSION_ROUTES, site_urls=SITE_A_URL)
+
+ assert result.raised is None
+ assert [(row["drive_id"], row["item_id"], row["id"]) for row in result.rows] == [
+ ("b!drive-a", "f1", "2.0"),
+ ("b!drive-a", "f1", "1.0"),
+ ("b!drive-a", "f3", "1.0"),
+ ]
+ assert all(row["site_id"] == SITE_A for row in result.rows)
+ assert all("@microsoft.graph.downloadUrl" not in row for row in result.rows)
+ assert [url for url in result.urls if "/versions" in url] == [
+ _g("/drives/b!drive-a/items/f1/versions"),
+ _g("/drives/b!drive-a/items/f1/versions?$skiptoken=2"),
+ _g("/drives/b!drive-a/items/f2/versions"),
+ _g("/drives/b!drive-a/items/f3/versions"),
+ ]
+
+ def test_failure_mid_page_resumes_at_the_page_in_progress(self) -> None:
+ failing = {**DRIVE_VERSION_ROUTES, _g("/drives/b!drive-a/items/f3/versions"): 403}
+
+ failed = _run("drive_item_versions", failing, site_urls=SITE_A_URL)
+
+ page_2 = _g("/drives/b!drive-a/root/delta?token=2")
+ assert isinstance(failed.raised, requests.HTTPError)
+ assert failed.final_state == SharePointResumeConfig(site_id=SITE_A, parent_id="b!drive-a", next_url=page_2)
+
+ resumed = _run(
+ "drive_item_versions", DRIVE_VERSION_ROUTES, site_urls=SITE_A_URL, resume_state=failed.final_state
+ )
+
+ assert resumed.raised is None
+ assert _g("/drives/b!drive-a/root/delta") not in resumed.urls
+ assert [(row["item_id"], row["id"]) for row in resumed.rows] == [("f1", "2.0"), ("f1", "1.0"), ("f3", "1.0")]
+
+ def test_list_item_versions_skip_hidden_lists_and_expand_fields(self) -> None:
+ routes: dict[str, Route] = {
+ **SITE_A_ROUTE,
+ _g(f"/sites/{SITE_A}/lists"): {
+ "value": [{"id": "list-1", "list": {"hidden": False}}, {"id": "hidden-list", "list": {"hidden": True}}]
+ },
+ _g(f"/sites/{SITE_A}/lists/list-1/items"): {"value": [{"id": "1"}, {"id": "2"}]},
+ _g(f"/sites/{SITE_A}/lists/list-1/items/1/versions"): {
+ "value": [
+ {"id": "2.0", "lastModifiedDateTime": "2026-02-01T00:00:00Z", "fields": {"Title": "Widget v2"}},
+ {"id": "1.0", "lastModifiedDateTime": "2026-01-01T00:00:00Z", "fields": {"Title": "Widget"}},
+ ]
+ },
+ _g(f"/sites/{SITE_A}/lists/list-1/items/2/versions"): {"value": []},
+ }
+
+ result = _run("list_item_versions", routes, site_urls=SITE_A_URL)
+
+ assert result.raised is None
+ assert [(row["list_id"], row["item_id"], row["id"], row["fields"]["Title"]) for row in result.rows] == [
+ ("list-1", "1", "2.0", "Widget v2"),
+ ("list-1", "1", "1.0", "Widget"),
+ ]
+ version_requests = [request for request in result.requests if request.path.endswith("/versions")]
+ assert [request.param("expand") for request in version_requests] == ["fields", "fields"]
+ assert all("hidden-list" not in url for url in result.urls)
+
+
class TestValidateCredentials:
@parameterized.expand(
[