LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3414 [FIX] SharePoint site_url/drive_id persistence and safe OAuth token refresh (#1919)
Kirtiman Mishra78fc0952cec9
diff --git a/backend/connector_auth_v2/constants.py b/backend/connector_auth_v2/constants.pyindex 886968d87..2508df7ed 100644--- a/backend/connector_auth_v2/constants.py+++ b/backend/connector_auth_v2/constants.py@@ -16,3 +16,21 @@ class SocialAuthConstants:GOOGLE_OAUTH = "google-oauth2"GOOGLE_TOKEN_EXPIRY_FORMAT = "%d/%m/%Y %H:%M:%S"+++# OAuth token-specific keys safe to merge across connectors sharing the same+# (provider, uid). Anything outside this set (form fields like site_url,+# drive_id, or provider-specific enrichment stored in ConnectorAuth.extra_data)+# must NOT leak between connectors.+OAUTH_TOKEN_KEYS: frozenset[str] = frozenset(+ {+ SocialAuthConstants.ACCESS_TOKEN,+ SocialAuthConstants.REFRESH_TOKEN,+ SocialAuthConstants.TOKEN_TYPE,+ SocialAuthConstants.EXPIRES,+ SocialAuthConstants.AUTH_TIME,+ SocialAuthConstants.REFRESH_AFTER,+ "expires_in",+ "scope",+ }+)diff --git a/backend/connector_auth_v2/models.py b/backend/connector_auth_v2/models.pyindex cd36e75cc..a92630c41 100644--- a/backend/connector_auth_v2/models.py+++ b/backend/connector_auth_v2/models.py@@ -10,7 +10,7 @@ from social_django.fields import JSONFieldfrom social_django.models import AbstractUserSocialAuth, DjangoStoragefrom social_django.strategy import DjangoStrategy-from connector_auth_v2.constants import SocialAuthConstants+from connector_auth_v2.constants import OAUTH_TOKEN_KEYS, SocialAuthConstantsfrom connector_auth_v2.pipeline.google import GoogleAuthHelperlogger = logging.getLogger(__name__)@@ -96,11 +96,27 @@ class ConnectorAuth(AbstractUserSocialAuth):refreshed_token = Truerelated_connector_instances = self.connectorinstance_set.all()for connector_instance in related_connector_instances:- connector_instance.connector_metadata = self.extra_data+ # Whitelist-merge: only OAuth token keys flow from the shared+ # extra_data into each sibling's metadata. Non-token keys (form+ # fields like site_url, drive_id, or provider-enrichment stored+ # in extra_data) must NOT leak between connectors sharing the+ # same (provider, uid).+ existing_metadata = connector_instance.connector_metadata or {}+ token_updates = {+ key: self.extra_data[key]+ for key in OAUTH_TOKEN_KEYS+ if self.extra_data.get(key) is not None+ }+ connector_instance.connector_metadata = {+ **existing_metadata,+ **token_updates,+ }connector_instance.save()logger.info(- f"Refreshed access token for connector {connector_instance.id}, "- f"provider: {self.provider}, uid: {self.uid}"+ "Refreshed access token for connector %s, provider: %s, uid: %s",+ connector_instance.id,+ self.provider,+ self.uid,)return self.extra_data, refreshed_tokendiff --git a/backend/connector_v2/fields.py b/backend/connector_v2/fields.pyindex 2a0f18c54..d3a65d91b 100644--- a/backend/connector_v2/fields.py+++ b/backend/connector_v2/fields.py@@ -1,7 +1,7 @@import loggingfrom datetime import datetime-from connector_auth_v2.constants import SocialAuthConstants+from connector_auth_v2.constants import OAUTH_TOKEN_KEYS, SocialAuthConstantsfrom connector_auth_v2.models import ConnectorAuthfrom django.db import models@@ -25,17 +25,28 @@ class ConnectorAuthJSONField(models.JSONField):refresh_after_str, SocialAuthConstants.REFRESH_AFTER_FORMAT)if datetime.now() > refresh_after:- metadata = self._refresh_tokens(provider, uid)+ metadata = self._refresh_tokens(provider, uid, metadata)return metadata- def _refresh_tokens(self, provider: str, uid: str) -> dict[str, str]:- """Retrieves PSA object and refreshes the token if necessary."""+ def _refresh_tokens(+ self, provider: str, uid: str, existing_metadata: dict[str, str]+ ) -> dict[str, str]:+ """Retrieves PSA object and refreshes the token if necessary.++ Whitelist-merges refreshed token fields over existing metadata so+ per-instance form fields (e.g. site_url, drive_id) are preserved on+ read and non-token keys from the shared ConnectorAuth.extra_data+ cannot leak into a connector's metadata.+ """connector_auth: ConnectorAuth = ConnectorAuth.get_social_auth(provider=provider, uid=uid)- if connector_auth:- (- connector_metadata,- _,- ) = connector_auth.get_and_refresh_tokens()- return connector_metadata # type: ignore+ if not connector_auth:+ return existing_metadata+ refreshed_metadata, _ = connector_auth.get_and_refresh_tokens()+ token_updates = {+ key: refreshed_metadata[key]+ for key in OAUTH_TOKEN_KEYS+ if refreshed_metadata.get(key) is not None+ }+ return {**existing_metadata, **token_updates}diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.pyindex 5517bc525..d639c923a 100644--- a/backend/connector_v2/serializers.py+++ b/backend/connector_v2/serializers.py@@ -2,12 +2,13 @@ import loggingfrom collections import OrderedDictfrom typing import Any+from connector_auth_v2.constants import OAUTH_TOKEN_KEYSfrom connector_auth_v2.models import ConnectorAuthfrom connector_auth_v2.pipeline.common import ConnectorAuthHelperfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeys-from connector_processor.exceptions import OAuthTimeOut-from rest_framework.serializers import CharField, SerializerMethodField+from connector_processor.exceptions import InvalidConnectorID, OAuthTimeOut+from rest_framework.serializers import CharField, SerializerMethodField, ValidationErrorfrom utils.fields import EncryptedBinaryFieldSerializerfrom utils.input_sanitizer import validate_name_field@@ -28,10 +29,57 @@ class ConnectorInstanceSerializer(AuditSerializer):class Meta:model = ConnectorInstancefields = "__all__"+ extra_kwargs = {"connector_name": {"required": False}}def validate_connector_name(self, value: str) -> str:return validate_name_field(value, field_name="Connector name")+ def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:+ """Backfill ``connector_name`` from the JSON schema default when absent.++ Defense-in-depth: the frontend RJSF form seeds ``connector_name`` from+ the schema default, but callers (including staging OAuth flows) have+ been observed to POST without it. If the connector schema declares a+ default name, use it. Otherwise raise a 400 explicitly rather than+ letting the missing value reach the DB and surface as an+ ``IntegrityError`` (the model enforces ``null=False``).++ Skipped entirely on partial updates (PATCH): the existing DB row+ already has a valid name, and backfilling would overwrite a+ user-renamed connector with the schema default.+ """+ attrs = super().validate(attrs)+ if attrs.get(CIKey.CONNECTOR_NAME) or self.partial:+ return attrs++ connector_id = attrs.get(CIKey.CONNECTOR_ID)+ default_name = (+ self._get_schema_default_connector_name(connector_id)+ if connector_id+ else None+ )+ if not default_name:+ raise ValidationError({CIKey.CONNECTOR_NAME: "This field is required."})+ attrs[CIKey.CONNECTOR_NAME] = default_name+ logger.info(+ "Filled missing connector_name with schema default for %s",+ connector_id,+ )+ return attrs++ @staticmethod+ def _get_schema_default_connector_name(connector_id: str) -> str | None:+ try:+ schema_details = ConnectorProcessor.get_json_schema(connector_id=connector_id)+ except InvalidConnectorID:+ return None+ return (+ schema_details.get(ConnectorKeys.JSON_SCHEMA, {})+ .get("properties", {})+ .get("connectorName", {})+ .get("default")+ )+def save(self, **kwargs): # type: ignoreuser = self.context.get("request").user or Noneconnector_id: str = kwargs[CIKey.CONNECTOR_ID]@@ -53,10 +101,22 @@ class ConnectorInstanceSerializer(AuditSerializer):oauth_credentials=kwargs[CIKey.CONNECTOR_METADATA],)kwargs[CIKey.CONNECTOR_AUTH] = connector_oauth- (- kwargs[CIKey.CONNECTOR_METADATA],- refresh_status,- ) = connector_oauth.get_and_refresh_tokens()+ # Merge refreshed token fields (whitelist) back into this+ # connector's metadata so ``super().save(**kwargs)`` does not+ # overwrite the fresh token the sibling-loop just persisted.+ # Whitelisting preserves per-connector form fields (site_url,+ # drive_id) that must not be leaked across connectors sharing+ # the same (provider, uid).+ refreshed_metadata, _ = connector_oauth.get_and_refresh_tokens()+ token_updates = {+ key: refreshed_metadata[key]+ for key in OAUTH_TOKEN_KEYS+ if refreshed_metadata.get(key) is not None+ }+ kwargs[CIKey.CONNECTOR_METADATA] = {+ **(kwargs.get(CIKey.CONNECTOR_METADATA) or {}),+ **token_updates,+ }except Exception as exc:logger.error("Error while obtaining ConnectorAuth for connector id "diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.pyindex 09a33be98..37525e4d5 100644--- a/backend/connector_v2/views.py+++ b/backend/connector_v2/views.py@@ -90,12 +90,17 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):# Only use OAuth flow if connector supports it AND oauth_key is providedif ConnectorInstance.supportsOAuth(connector_id=connector_id) and oauth_key:- connector_metadata = ConnectorAuthHelper.get_oauth_creds_from_cache(+ oauth_tokens = ConnectorAuthHelper.get_oauth_creds_from_cache(cache_key=oauth_key,delete_key=False, # Don't delete yet - wait for successful operation)- if connector_metadata is None:+ if oauth_tokens is None:raise MissingParamException(param=ConnectorAuthKey.OAUTH_KEY)+ # Preserve non-secret form fields (e.g. site_url connector Sharepoint)+ form_metadata = self.request.data.get(CIKey.CONNECTOR_METADATA) or {}+ if not isinstance(form_metadata, dict):+ form_metadata = {}+ connector_metadata = {**form_metadata, **oauth_tokens}else:connector_metadata = self.request.data.get(CIKey.CONNECTOR_METADATA)return connector_metadatadiff --git a/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/sharepoint.py b/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/sharepoint.pyindex 3eacdac30..e8acd44f3 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/sharepoint.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/sharepoint.py@@ -137,8 +137,8 @@ class SharePointFileSystem(AbstractFileSystem):ctx = self._get_context()if self.drive_id:- # Specific drive by ID- self._drive = ctx.drives.get_by_id(self.drive_id)+ # Specific drive by ID — EntityCollection uses bracket indexing.+ self._drive = ctx.drives[self.drive_id]elif self.site_url and "sharepoint.com" in self.site_url.lower():# SharePoint site - get default document libraryself._drive = self._get_sharepoint_site_drive(ctx)@@ -149,15 +149,15 @@ class SharePointFileSystem(AbstractFileSystem):return self._drivedef _get_sharepoint_site_drive(self, ctx: Any) -> Any:- """Get drive from SharePoint site URL."""+ """Get drive from SharePoint site URL.++ Uses the library's get_by_url, which maps an absolute site URL to the+ Graph API's ``/sites/{hostname}:/{server-relative-path}`` addressing.+ """from urllib.parse import urlparse- parsed = urlparse(self.site_url)- # Extract site path from URL like- # https://tenant.sharepoint.com/sites/sitename- site_path = parsed.path.rstrip("/")- if site_path:- return ctx.sites.get_by_path(site_path).drive+ if urlparse(self.site_url).path.strip("/"):+ return ctx.sites.get_by_url(self.site_url).drivereturn ctx.sites.root.drivedef _get_onedrive_drive(self, ctx: Any) -> Any:diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/static/json_schema.jsonindex d83f84c8a..e1f0757e3 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/sharepoint/static/json_schema.json@@ -22,6 +22,7 @@"drive_id": {"type": "string","title": "Drive ID",+ "format": "password","description": "Specific Drive/Document Library ID. Leave empty to use the default drive."},"auth_type": {@@ -64,7 +65,7 @@"type": "string","title": "User Email","format": "password",- "description": "User's email address. Required ONLY for OneDrive with Client Credentials (not needed for SharePoint).",+ "description": "Required only to access OneDrive with Client Credentials (e.g., [email protected]). Leave empty when accessing a SharePoint site via Site URL.","examples": [