RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(data_source): add Azure DevOps connector (#18715)
Erhan Aydın72b7b02113a8
diff --git a/common/constants.py b/common/constants.pyindex 51fb3c7e17..bdb1ccf208 100644--- a/common/constants.py+++ b/common/constants.py@@ -166,6 +166,7 @@ class FileSource(StrEnum):GITLAB = "gitlab"IMAP = "imap"BITBUCKET = "bitbucket"+ AZURE_DEVOPS = "azure_devops"ZENDESK = "zendesk"SEAFILE = "seafile"MYSQL = "mysql"diff --git a/common/data_source/__init__.py b/common/data_source/__init__.pyindex 6d833c8332..165769ef26 100644--- a/common/data_source/__init__.py+++ b/common/data_source/__init__.py@@ -30,6 +30,7 @@ from .airtable_connector import AirtableConnectorfrom .asana_connector import AsanaConnectorfrom .azure_blob_connector import AzureBlobConnectorfrom .bigquery_connector import BigQueryConnector+from .azure_devops.connector import AzureDevOpsConnectorfrom .bitbucket.connector import BitbucketConnectorfrom .blob_connector import BlobStorageConnectorfrom .box_connector import BoxConnector@@ -93,6 +94,7 @@ CONNECTOR_BY_SOURCE: dict[str, type] = {FileSource.ZENDESK: ZendeskConnector,FileSource.GITLAB: GitlabConnector,FileSource.BITBUCKET: BitbucketConnector,+ FileSource.AZURE_DEVOPS: AzureDevOpsConnector,FileSource.SEAFILE: SeaFileConnector,FileSource.DINGTALK_AI_TABLE: DingTalkAITableConnector,FileSource.MYSQL: RDBMSConnector,@@ -132,6 +134,7 @@ __all__ = ["GithubConnector","GitlabConnector","BitbucketConnector",+ "AzureDevOpsConnector","BoxConnector","OneDriveConnector","OutlookConnector",diff --git a/common/data_source/azure_devops/__init__.py b/common/data_source/azure_devops/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/common/data_source/azure_devops/connector.py b/common/data_source/azure_devops/connector.pynew file mode 100644index 0000000000..b20e2adf0f--- /dev/null+++ b/common/data_source/azure_devops/connector.py@@ -0,0 +1,650 @@+from __future__ import annotations++import copy+import logging+from collections.abc import Iterator+from datetime import datetime, timezone+from typing import Any, TYPE_CHECKING++from typing_extensions import override++from common.data_source.azure_devops.utils import (+ build_auth_client,+ code_document_id,+ default_branch_of,+ fetch_file_content,+ fetch_pull_request,+ get_json,+ list_items,+ list_projects,+ list_repositories,+ map_item_to_document,+ map_pull_request_to_document,+ organization_url,+ pull_request_document_id,+ pull_request_in_window,+ pull_request_may_be_truncated,+ raise_for_auth,+ API_VERSION,+)+from common.data_source.config import INDEX_BATCH_SIZE+from common.data_source.exceptions import (+ ConnectorMissingCredentialError,+ CredentialExpiredError,+ InsufficientPermissionsError,+ UnexpectedValidationError,+)+from common.data_source.interfaces import (+ CheckpointedConnector,+ CheckpointOutput,+ IndexingHeartbeatInterface,+ SecondsSinceUnixEpoch,+ SlimConnectorWithPermSync,+)+from common.data_source.models import ConnectorCheckpoint, ConnectorFailure, DocumentFailure, SlimDocument++if TYPE_CHECKING:+ import httpx++# Which repositories to index. Mirrors the scope selector other Git connectors use.+INDEX_MODE_ORGANIZATION = "organization"+INDEX_MODE_PROJECTS = "projects"+INDEX_MODE_REPOSITORIES = "repositories"++# Which document families to produce.+CONTENT_CODE = "code"+CONTENT_PULL_REQUESTS = "pull_requests"+CONTENT_BOTH = "both"++STAGE_CODE = "code"+STAGE_PULL_REQUESTS = "pull_requests"++# Files handled per invocation before the checkpoint is persisted. Large+# monorepos would otherwise run for a long time with no resumable progress.+MAX_FILES_PER_CALL = 500+PR_PAGE_SIZE = 100+++class AzureDevOpsConnectorCheckpoint(ConnectorCheckpoint):+ """Checkpoint state for resumable Azure DevOps indexing.++ Fields:+ repos_queue: Repositories to visit, each as ``{project, name, branch}``.+ current_repo_index: Repository currently being processed.+ stage: Which document family is being indexed for that repository.+ file_offset: Fast lookup hint into the repository file listing.+ pr_skip: Pagination hint for pull requests.+ last_source_id: Anchor identifying the last item handled in this stage.+ retry_paths: Files that failed once and are retried before the stage ends.++ Offsets are positions in a remote listing that shifts whenever a file or a+ pull request is added or removed, so resuming on an offset alone can skip or+ repeat an item. The anchor decides where the walk continues; the offset only+ makes finding it cheap.+ """++ repos_queue: list[dict[str, str]] = []+ current_repo_index: int = 0+ stage: str = STAGE_CODE+ file_offset: int = 0+ pr_skip: int = 0+ last_source_id: str | None = None+ retry_paths: list[str] = []+++class AzureDevOpsConnector(+ CheckpointedConnector[AzureDevOpsConnectorCheckpoint],+ SlimConnectorWithPermSync,+):+ """Connector for indexing Azure DevOps (Azure Repos) source files and pull requests.++ Works against both Azure DevOps Services and self-hosted Azure DevOps Server:+ pass a bare organization name for the former, or the full collection URL for+ the latter.++ Args:+ organization: Organization name, or base URL of a self-hosted collection.+ index_mode: Scope selector — ``organization``, ``projects`` or ``repositories``.+ projects: Comma-separated team projects, used when ``index_mode`` is ``projects``.+ repositories: Comma-separated repositories, used when ``index_mode`` is+ ``repositories``. Entries may be ``project/repo`` or a bare repository name.+ content_types: ``code``, ``pull_requests`` or ``both``.+ batch_size: Max number of slim documents to yield per batch.+ """++ def __init__(+ self,+ organization: str,+ index_mode: str = INDEX_MODE_ORGANIZATION,+ projects: str | None = None,+ repositories: str | None = None,+ content_types: str = CONTENT_BOTH,+ batch_size: int = INDEX_BATCH_SIZE,+ ) -> None:+ self.organization = organization+ self.index_mode = index_mode or INDEX_MODE_ORGANIZATION+ self._projects = self._split(projects)+ self._repositories = self._split(repositories)+ self.content_types = content_types or CONTENT_BOTH+ self.batch_size = batch_size+ self.personal_access_token: str | None = None++ @staticmethod+ def _split(value: str | None) -> list[str] | None:+ items = [part.strip() for part in value.split(",")] if value else []+ return [item for item in items if item] or None++ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "AzureDevOpsConnector":+ credentials = config.get("credentials") or {}+ connector = cls(+ organization=config.get("organization"),+ index_mode=config.get("index_mode") or INDEX_MODE_ORGANIZATION,+ projects=config.get("projects"),+ repositories=config.get("repositories"),+ content_types=config.get("content_types") or CONTENT_BOTH,+ )+ connector.load_credentials({"azure_devops_pat": credentials.get("azure_devops_pat")})+ return connector++ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:+ """Load the personal access token. Requires the ``Code (Read)`` scope."""+ self.personal_access_token = credentials.get("azure_devops_pat")+ if not self.personal_access_token:+ raise ConnectorMissingCredentialError("Azure DevOps")+ return None++ @property+ def _org_url(self) -> str:+ return organization_url(self.organization)++ def _client(self) -> "httpx.Client":+ if not self.personal_access_token:+ raise ConnectorMissingCredentialError("Azure DevOps")+ return build_auth_client(self.personal_access_token)++ def _repo_api_url(self, project: str, repo_name: str) -> str:+ return f"{self._org_url}/{project}/_apis/git/repositories/{repo_name}"++ def _indexes_code(self) -> bool:+ return self.content_types in (CONTENT_CODE, CONTENT_BOTH)++ def _indexes_pull_requests(self) -> bool:+ return self.content_types in (CONTENT_PULL_REQUESTS, CONTENT_BOTH)++ def _matches_repository_filter(self, project: str, repo_name: str) -> bool:+ """Repository entries may be ``project/repo`` or a bare repository name.++ Azure DevOps repository names are unique per project, not per+ organization, so the qualified form is the unambiguous one.+ """+ if self.index_mode != INDEX_MODE_REPOSITORIES or not self._repositories:+ return True+ return repo_name in self._repositories or f"{project}/{repo_name}" in self._repositories++ def _discover_repositories(self, client: "httpx.Client") -> list[dict[str, str]]:+ """Resolve the repositories to index.++ Without an explicit project list the organization-scoped endpoint returns+ every repository in one request; otherwise each project is queried.+ """+ if self.index_mode == INDEX_MODE_PROJECTS and self._projects:+ scopes: list[str | None] = list(self._projects)+ elif self.index_mode == INDEX_MODE_REPOSITORIES and self._repositories:+ qualified = {entry.split("/", 1)[0] for entry in self._repositories if "/" in entry}+ scopes = sorted(qualified) if qualified else [None]+ else:+ scopes = [None]++ discovered: list[dict[str, str]] = []++ for scope in scopes:+ for repo in list_repositories(client, self._org_url, scope):+ name = repo.get("name")+ project = (repo.get("project") or {}).get("name") or scope or ""+ if not name or not project:+ continue+ if not self._matches_repository_filter(project, name):+ continue+ discovered.append({"project": project, "name": name, "branch": default_branch_of(repo)})++ return sorted(discovered, key=lambda repo: (repo["project"], repo["name"]))++ def _iter_pull_request_page(self, client: "httpx.Client", project: str, repo_name: str, skip: int) -> list[dict[str, Any]]:+ payload = get_json(+ client,+ f"{self._repo_api_url(project, repo_name)}/pullrequests",+ {"searchCriteria.status": "all", "$top": PR_PAGE_SIZE, "$skip": skip},+ )+ return payload.get("value") or []++ @override+ def load_from_checkpoint(+ self,+ start: SecondsSinceUnixEpoch,+ end: SecondsSinceUnixEpoch,+ checkpoint: AzureDevOpsConnectorCheckpoint,+ ) -> CheckpointOutput[AzureDevOpsConnectorCheckpoint]:+ """Resumable ingestion across repositories, files and pull requests.++ One repository stage is processed per invocation; the returned checkpoint+ records the exact resume position.+ """+ new_checkpoint = copy.deepcopy(checkpoint)+ window_start = datetime.fromtimestamp(start, tz=timezone.utc)+ window_end = datetime.fromtimestamp(end, tz=timezone.utc)++ with self._client() as client:+ if not new_checkpoint.repos_queue:+ new_checkpoint.repos_queue = self._discover_repositories(client)+ logging.info(+ "[AzureDevOps] discovered %d repositories (organization=%s, index_mode=%s, content_types=%s)",+ len(new_checkpoint.repos_queue),+ self.organization,+ self.index_mode,+ self.content_types,+ )+ new_checkpoint.current_repo_index = 0+ new_checkpoint.stage = STAGE_CODE+ new_checkpoint.file_offset = 0+ new_checkpoint.pr_skip = 0++ repos = new_checkpoint.repos_queue+ if not repos or new_checkpoint.current_repo_index >= len(repos):+ new_checkpoint.has_more = False+ return new_checkpoint++ repo = repos[new_checkpoint.current_repo_index]+ project, repo_name, branch = repo["project"], repo["name"], repo["branch"]+ logging.info(+ "[AzureDevOps] %s/%s stage=%s file_offset=%d pr_skip=%d (repo %d/%d)",+ project,+ repo_name,+ new_checkpoint.stage,+ new_checkpoint.file_offset,+ new_checkpoint.pr_skip,+ new_checkpoint.current_repo_index + 1,+ len(repos),+ )++ if new_checkpoint.stage == STAGE_CODE:+ if self._indexes_code():+ yield from self._load_code(client, new_checkpoint, project, repo_name, branch, window_start, window_end)+ else:+ new_checkpoint.stage = STAGE_PULL_REQUESTS+ elif new_checkpoint.stage == STAGE_PULL_REQUESTS:+ if self._indexes_pull_requests():+ yield from self._load_pull_requests(client, new_checkpoint, project, repo_name, window_start, window_end)+ else:+ self._advance_repo(new_checkpoint)++ new_checkpoint.has_more = new_checkpoint.current_repo_index < len(new_checkpoint.repos_queue)++ return new_checkpoint++ def _resolve_file_start(+ self,+ items: list[dict[str, Any]],+ checkpoint: AzureDevOpsConnectorCheckpoint,+ project: str,+ repo_name: str,+ ) -> int:+ """Position the file walk just after the anchored item.++ The stored offset is checked first because it is right in the common+ case; otherwise the listing is searched. Only when the anchor is gone —+ the file was deleted or renamed — does the offset stand on its own.+ """+ anchor = checkpoint.last_source_id+ if not anchor:+ return checkpoint.file_offset++ def source_id_of(item: dict[str, Any]) -> str:+ return code_document_id(self.organization, project, repo_name, item.get("path", "").lstrip("/"))++ hint = checkpoint.file_offset+ if 0 < hint <= len(items) and source_id_of(items[hint - 1]) == anchor:+ return hint++ for index, item in enumerate(items):+ if source_id_of(item) == anchor:+ return index + 1++ logging.warning(+ "[AzureDevOps] %s/%s resume anchor %s is gone from the listing; falling back to offset %d",+ project,+ repo_name,+ anchor,+ checkpoint.file_offset,+ )+ return checkpoint.file_offset++ def _filter_resumed_pull_requests(+ self,+ pull_requests: list[dict[str, Any]],+ checkpoint: AzureDevOpsConnectorCheckpoint,+ project: str,+ repo_name: str,+ ) -> list[dict[str, Any]]:+ """Drop the pull requests already committed, using the anchor."""+ anchor = checkpoint.last_source_id+ if not anchor:+ return pull_requests++ for index, pull_request in enumerate(pull_requests):+ source_id = pull_request_document_id(self.organization, project, repo_name, pull_request.get("pullRequestId"))+ if source_id == anchor:+ return pull_requests[index + 1 :]+ return pull_requests++ def _load_code(+ self,+ client: "httpx.Client",+ checkpoint: AzureDevOpsConnectorCheckpoint,+ project: str,+ repo_name: str,+ branch: str,+ window_start: datetime,+ window_end: datetime,+ ) -> Iterator[Any]:+ """Yield source files, continuing after the anchored item."""+ repo_api_url = self._repo_api_url(project, repo_name)+ items = list_items(client, repo_api_url, branch)+ start = self._resolve_file_start(items, checkpoint, project, repo_name)+ window = items[start : start + MAX_FILES_PER_CALL]++ for item in window:+ path = item.get("path", "")+ source_id = code_document_id(self.organization, project, repo_name, path.lstrip("/"))+ try:+ yield from self._emit_file(client, repo_api_url, item, project, repo_name, branch, window_start, window_end)+ except Exception as e:+ # The item is queued for one retry before the stage ends, so a+ # transient failure does not quietly drop the file from the run.+ if path not in checkpoint.retry_paths:+ checkpoint.retry_paths.append(path)+ yield self._file_failure(path, source_id, project, repo_name, e)+ finally:+ checkpoint.last_source_id = source_id++ checkpoint.file_offset = start + len(window)+ logging.info(+ "[AzureDevOps] %s/%s indexed files %d/%d",+ project,+ repo_name,+ checkpoint.file_offset,+ len(items),+ )++ if checkpoint.file_offset < len(items):+ return++ if checkpoint.retry_paths:+ yield from self._retry_failed_files(client, repo_api_url, items, checkpoint, project, repo_name, branch, window_start, window_end)++ checkpoint.file_offset = 0+ checkpoint.last_source_id = None+ if self._indexes_pull_requests():+ checkpoint.stage = STAGE_PULL_REQUESTS+ else:+ self._advance_repo(checkpoint)++ def _emit_file(+ self,+ client: "httpx.Client",+ repo_api_url: str,+ item: dict[str, Any],+ project: str,+ repo_name: str,+ branch: str,+ window_start: datetime,+ window_end: datetime,+ ) -> Iterator[Any]:+ """Yield the document for one file, or nothing when it is out of scope."""+ path = item.get("path", "")+ change = item.get("latestProcessedChange") or {}+ committer = change.get("committer") or change.get("author") or {}+ changed_at = committer.get("date")++ if isinstance(changed_at, str):+ changed = datetime.fromisoformat(changed_at.replace("Z", "+00:00")).astimezone(timezone.utc)+ if not (window_start <= changed <= window_end):+ return++ content = fetch_file_content(client, repo_api_url, path, branch)+ if content is None:+ return++ yield map_item_to_document(item, content, self.organization, self._org_url, project, repo_name, branch)++ def _file_failure(self, path: str, source_id: str, project: str, repo_name: str, error: Exception) -> ConnectorFailure:+ return ConnectorFailure(+ failed_document=DocumentFailure(+ document_id=source_id,+ document_link=f"{self._org_url}/{project}/_git/{repo_name}?path={path}",+ ),+ failure_message=f"Failed to process Azure DevOps file {path}: {error}",+ exception=error,+ )++ def _retry_failed_files(+ self,+ client: "httpx.Client",+ repo_api_url: str,+ items: list[dict[str, Any]],+ checkpoint: AzureDevOpsConnectorCheckpoint,+ project: str,+ repo_name: str,+ branch: str,+ window_start: datetime,+ window_end: datetime,+ ) -> Iterator[Any]:+ """Re-attempt the files that failed earlier in this repository, once.++ Advancing past a failed file without another attempt would drop it from+ the run entirely. Retrying exactly once keeps a deterministic failure+ from stalling the sync.+ """+ pending, checkpoint.retry_paths = checkpoint.retry_paths, []+ by_path = {item.get("path", ""): item for item in items}+ logging.info("[AzureDevOps] %s/%s retrying %d failed file(s)", project, repo_name, len(pending))++ for path in pending:+ item = by_path.get(path)+ if item is None:+ continue+ try:+ yield from self._emit_file(client, repo_api_url, item, project, repo_name, branch, window_start, window_end)+ except Exception as e:+ source_id = code_document_id(self.organization, project, repo_name, path.lstrip("/"))+ yield self._file_failure(path, source_id, project, repo_name, e)++ def _load_pull_requests(+ self,+ client: "httpx.Client",+ checkpoint: AzureDevOpsConnectorCheckpoint,+ project: str,+ repo_name: str,+ window_start: datetime,+ window_end: datetime,+ ) -> Iterator[Any]:+ """Yield one page of pull requests, resuming from ``checkpoint.pr_skip``.++ Azure DevOps has no reliable "updated since" filter on this endpoint, so+ the time window is applied client-side.+ """+ page = self._iter_pull_request_page(client, project, repo_name, checkpoint.pr_skip)+ page_size = len(page)+ pull_requests = self._filter_resumed_pull_requests(page, checkpoint, project, repo_name)++ for pull_request in pull_requests:+ pr_id = pull_request.get("pullRequestId")+ try:+ if not pull_request_in_window(pull_request, window_start, window_end):+ continue+ if pull_request_may_be_truncated(pull_request):+ pull_request = fetch_pull_request(client, self._repo_api_url(project, repo_name), pr_id)+ yield map_pull_request_to_document(pull_request, self.organization, self._org_url, project, repo_name)+ except Exception as e:+ yield ConnectorFailure(+ failed_document=DocumentFailure(+ document_id=pull_request_document_id(self.organization, project, repo_name, pr_id),+ document_link=f"{self._org_url}/{project}/_git/{repo_name}/pullrequest/{pr_id}",+ ),+ failure_message=f"Failed to process Azure DevOps pull request {pr_id}: {e}",+ exception=e,+ )+ finally:+ checkpoint.last_source_id = pull_request_document_id(self.organization, project, repo_name, pr_id)++ if page_size < PR_PAGE_SIZE:+ self._advance_repo(checkpoint)+ else:+ checkpoint.pr_skip += PR_PAGE_SIZE++ @staticmethod+ def _advance_repo(checkpoint: AzureDevOpsConnectorCheckpoint) -> None:+ checkpoint.current_repo_index += 1+ checkpoint.stage = STAGE_CODE+ checkpoint.file_offset = 0+ checkpoint.pr_skip = 0+ checkpoint.last_source_id = None+ checkpoint.retry_paths = []++ @override+ def build_dummy_checkpoint(self) -> AzureDevOpsConnectorCheckpoint:+ """Create an initial checkpoint with work remaining."""+ return AzureDevOpsConnectorCheckpoint(has_more=True)++ @override+ def validate_checkpoint_json(self, checkpoint_json: str) -> AzureDevOpsConnectorCheckpoint:+ """Validate and deserialize a checkpoint instance from JSON."""+ return AzureDevOpsConnectorCheckpoint.model_validate_json(checkpoint_json)++ def retrieve_all_slim_docs_perm_sync(+ self,+ callback: IndexingHeartbeatInterface | None = None,+ ) -> Iterator[list[SlimDocument]]:+ """Return document ids only, for pruning deleted files and pull requests."""+ batch: list[SlimDocument] = []++ with self._client() as client:+ for repo in self._discover_repositories(client):+ project, repo_name, branch = repo["project"], repo["name"], repo["branch"]+ repo_api_url = self._repo_api_url(project, repo_name)++ if self._indexes_code():+ for item in list_items(client, repo_api_url, branch):+ path = item.get("path", "").lstrip("/")+ batch.append(SlimDocument(id=code_document_id(self.organization, project, repo_name, path)))+ if len(batch) >= self.batch_size:+ yield batch+ emitted = len(batch)+ batch = []+ if callback:+ if callback.should_stop():+ raise RuntimeError("azure_devops_sync: Stop signal detected")+ callback.progress("azure_devops_sync", emitted)++ if self._indexes_pull_requests():+ skip = 0+ while True:+ pull_requests = self._iter_pull_request_page(client, project, repo_name, skip)+ for pull_request in pull_requests:+ batch.append(SlimDocument(id=pull_request_document_id(self.organization, project, repo_name, pull_request.get("pullRequestId"))))+ if len(batch) >= self.batch_size:+ yield batch+ emitted = len(batch)+ batch = []+ if callback:+ if callback.should_stop():+ raise RuntimeError("azure_devops_sync: Stop signal detected")+ callback.progress("azure_devops_sync", emitted)+ if len(pull_requests) < PR_PAGE_SIZE:+ break+ skip += PR_PAGE_SIZE++ if batch:+ yield batch++ def _validate_settings(self) -> None:+ """Reject unusable configuration before any request is made.++ An unknown selector would otherwise pass silently and the sync would+ complete without producing a single document.+ """+ if self.index_mode not in (INDEX_MODE_ORGANIZATION, INDEX_MODE_PROJECTS, INDEX_MODE_REPOSITORIES):+ raise UnexpectedValidationError(f"Unsupported index mode: {self.index_mode}")+ if self.content_types not in (CONTENT_CODE, CONTENT_PULL_REQUESTS, CONTENT_BOTH):+ raise UnexpectedValidationError(f"Unsupported content types: {self.content_types}")+ if self.index_mode == INDEX_MODE_PROJECTS and not self._projects:+ raise UnexpectedValidationError("At least one team project is required when indexing by project.")+ if self.index_mode == INDEX_MODE_REPOSITORIES and not self._repositories:+ raise UnexpectedValidationError("At least one repository is required when indexing by repository.")++ def validate_connector_settings(self) -> None:+ """Probe a lightweight endpoint to verify credentials and organization access.++ Raises:+ CredentialExpiredError: on HTTP 401, or on the HTTP 203 sign-in page+ Azure DevOps returns for an invalid token.+ InsufficientPermissionsError: on HTTP 403.+ UnexpectedValidationError: on any other failure.+ """+ # Settings are checked before the remote probe: an unusable configuration+ # should fail immediately rather than after a network round trip.+ self._validate_settings()++ try:+ with self._client() as client:+ response = client.get(+ f"{self._org_url}/_apis/projects",+ params={"api-version": API_VERSION, "$top": 1},+ )+ raise_for_auth(response)+ if response.status_code == 404:+ raise UnexpectedValidationError(f"Azure DevOps organization not found: {self.organization}")+ if response.status_code < 200 or response.status_code >= 300:+ raise UnexpectedValidationError(f"Unexpected Azure DevOps error (status={response.status_code}).")++ if self._projects:+ known = set(list_projects(client, self._org_url))+ missing = [project for project in self._projects if project not in known]+ if missing:+ raise UnexpectedValidationError(f"Team project(s) not found or not visible: {', '.join(missing)}")+ except Exception as e:+ if isinstance(+ e,+ (+ CredentialExpiredError,+ InsufficientPermissionsError,+ UnexpectedValidationError,+ ConnectorMissingCredentialError,+ ),+ ):+ raise+ raise UnexpectedValidationError(f"Unexpected error while validating Azure DevOps settings: {e}")+++if __name__ == "__main__":+ connector = AzureDevOpsConnector(organization="<YOUR_ORGANIZATION>", index_mode=INDEX_MODE_PROJECTS, projects="<YOUR_PROJECT>")+ connector.load_credentials({"azure_devops_pat": "<YOUR_PAT>"})+ connector.validate_connector_settings()+ print("Credentials validated successfully.")++ azure_checkpoint = connector.build_dummy_checkpoint()+ start_time = datetime.fromtimestamp(0, tz=timezone.utc)+ end_time = datetime.now(timezone.utc)++ while azure_checkpoint.has_more:+ generator = connector.load_from_checkpoint(+ start=start_time.timestamp(),+ end=end_time.timestamp(),+ checkpoint=azure_checkpoint,+ )+ while True:+ try:+ print(next(generator))+ except StopIteration as stop:+ azure_checkpoint = stop.value+ breakdiff --git a/common/data_source/azure_devops/utils.py b/common/data_source/azure_devops/utils.pynew file mode 100644index 0000000000..aaaa800f8d--- /dev/null+++ b/common/data_source/azure_devops/utils.py@@ -0,0 +1,495 @@+from __future__ import annotations++import logging+import time+from collections.abc import Iterator+from datetime import datetime, timezone+from typing import Any+from urllib.parse import quote++import httpx++from common.data_source.config import REQUEST_TIMEOUT_SECONDS, DocumentSource+from common.data_source.cross_connector_utils.rate_limit_wrapper import rate_limit_builder+from common.data_source.cross_connector_utils.retry_wrapper import retry_builder+from common.data_source.exceptions import (+ CredentialExpiredError,+ InsufficientPermissionsError,+ UnexpectedValidationError,+)+from common.data_source.models import BasicExpertInfo, Document+from common.data_source.utils import get_file_ext++API_VERSION = "7.1"++# Azure DevOps returns binary blobs for these; indexing them adds noise, not signal.+BINARY_EXTENSIONS: frozenset[str] = frozenset(+ {+ ".png",+ ".jpg",+ ".jpeg",+ ".gif",+ ".bmp",+ ".ico",+ ".svg",+ ".webp",+ ".pdf",+ ".zip",+ ".gz",+ ".tar",+ ".7z",+ ".rar",+ ".jar",+ ".war",+ ".dll",+ ".exe",+ ".so",+ ".dylib",+ ".pdb",+ ".class",+ ".pyc",+ ".woff",+ ".woff2",+ ".ttf",+ ".eot",+ ".otf",+ ".mp3",+ ".mp4",+ ".avi",+ ".mov",+ ".psd",+ ".xlsx",+ ".docx",+ }+)++# Directories that are build output or vendored dependencies.+EXCLUDED_PATH_SEGMENTS: tuple[str, ...] = (+ "/node_modules/",+ "/bin/",+ "/obj/",+ "/dist/",+ "/build/",+ "/target/",+ "/vendor/",+ "/packages/",+ "/.git/",+ "/__pycache__/",+ "/.venv/",+)++# Version-control metadata: present in every repository, no retrievable signal.+SKIPPED_FILENAMES: frozenset[str] = frozenset({".gitattributes", ".gitignore", ".gitkeep", ".gitmodules", ".dockerignore", ".editorconfig"})++MAX_FILE_BYTES = 1_000_000++# The pull request *list* endpoint truncates descriptions at 400 characters;+# only the single pull request endpoint returns the full text.+PULL_REQUEST_DESCRIPTION_LIMIT = 400++# Upper bound for a server supplied Retry-After. Azure DevOps can ask for a long+# pause, and honouring it verbatim would park a sync worker for hours.+MAX_RETRY_AFTER_SECONDS = 30+++def build_auth_client(personal_access_token: str) -> httpx.Client:+ """Create an authenticated client for the Azure DevOps REST API.++ Azure DevOps expects the PAT as the password of a basic-auth pair with an+ empty username.+ """+ return httpx.Client(auth=("", personal_access_token), timeout=REQUEST_TIMEOUT_SECONDS)+++def organization_url(organization: str) -> str:+ """Resolve the API root for a hosted organization or a self-hosted server.++ ``organization`` may be a bare organization name (Azure DevOps Services) or a+ full base URL such as ``https://tfs.contoso.com/DefaultCollection`` for+ Azure DevOps Server.+ """+ if organization.startswith("http://"):+ raise UnexpectedValidationError("Azure DevOps collection URLs must use HTTPS; the personal access token is sent in the Authorization header.")+ if organization.startswith("https://"):+ return organization.rstrip("/")+ return f"https://dev.azure.com/{quote(organization, safe='')}"+++def raise_for_auth(response: httpx.Response, expect_json: bool = True) -> None:+ """Translate Azure DevOps auth failures into connector errors.++ Azure DevOps does not answer an invalid or unauthorized PAT with 401. It+ answers **203 Non-Authoritative Information** and returns the HTML sign-in+ page, so a naive ``raise_for_status`` succeeds and the JSON parse fails+ later with an unrelated error. Detect it here instead.++ The HTML heuristic only applies to endpoints that return JSON: a repository+ can legitimately contain ``.html`` files, and those must not be mistaken for+ a sign-in page.+ """+ if response.status_code == 203:+ raise CredentialExpiredError("Invalid or expired Azure DevOps personal access token (HTTP 203 sign-in page).")+ if expect_json and "text/html" in response.headers.get("content-type", ""):+ raise CredentialExpiredError("Azure DevOps returned a sign-in page; the personal access token is invalid or unauthorized.")+ if response.status_code == 401:+ raise CredentialExpiredError("Invalid or expired Azure DevOps personal access token (HTTP 401).")+ if response.status_code == 403:+ raise InsufficientPermissionsError("Personal access token lacks the required 'Code (Read)' scope (HTTP 403).")+++def sleep_for_retry_after(retry_after: str | None) -> None:+ """Honour a ``Retry-After`` header, clamped to the retry backoff ceiling."""+ if retry_after is None:+ return+ try:+ seconds = int(retry_after)+ except (TypeError, ValueError):+ return+ time.sleep(max(0, min(seconds, MAX_RETRY_AFTER_SECONDS)))+++class AzureDevOpsRetriableError(Exception):+ """Raised for throttling and server-side failures worth retrying."""+++class AzureDevOpsNonRetriableError(Exception):+ """Raised for client errors that retrying cannot fix."""+++@retry_builder(+ tries=6,+ delay=1,+ backoff=2,+ max_delay=30,+ exceptions=(AzureDevOpsRetriableError, httpx.RequestError),+)+@rate_limit_builder(max_calls=120, period=60)+def azure_devops_get(+ client: httpx.Client,+ url: str,+ params: dict[str, Any] | None = None,+ expect_json: bool = True,+) -> httpx.Response:+ """Perform a GET against Azure DevOps with retry and rate limiting.++ Azure DevOps throttles on consumed throughput units and answers with 429+ plus a ``Retry-After`` header; a full organization crawl touches every+ repository, so honouring it matters. Authentication failures are mapped+ first and deliberately left non-retriable.+ """+ response = client.get(url, params=params, timeout=REQUEST_TIMEOUT_SECONDS)+ raise_for_auth(response, expect_json=expect_json)++ status = response.status_code+ if status == 429:+ sleep_for_retry_after(response.headers.get("Retry-After"))+ raise AzureDevOpsRetriableError("Azure DevOps rate limit exceeded (429).")+ if 500 <= status < 600:+ raise AzureDevOpsRetriableError(f"Azure DevOps server error: {status}")+ if 400 <= status < 500:+ raise AzureDevOpsNonRetriableError(f"Azure DevOps client error: {status} for {url}")++ return response+++def get_json(client: httpx.Client, url: str, params: dict[str, Any] | None = None) -> dict[str, Any]:+ """GET a JSON payload, mapping auth and HTTP failures to connector errors."""+ query: dict[str, Any] = {"api-version": API_VERSION}+ if params:+ query.update(params)++ response = azure_devops_get(client, url, query)+ if response.status_code < 200 or response.status_code >= 300:+ raise UnexpectedValidationError(f"Azure DevOps request failed (status={response.status_code}) for {url}")+ return response.json()+++def list_projects(client: httpx.Client, org_url: str) -> Iterator[str]:+ """Yield every team project name visible to the token."""+ payload = get_json(client, f"{org_url}/_apis/projects", {"$top": 1000})+ for project in payload.get("value") or []:+ name = project.get("name")+ if name:+ yield name+++def list_repositories(client: httpx.Client, org_url: str, project: str | None) -> Iterator[dict[str, Any]]:+ """Yield repositories for one project, or for the whole organization.++ The organization-scoped form returns repositories across every project in a+ single request, which keeps repository discovery to one round trip.+ """+ scope = f"{org_url}/{quote(project, safe='')}" if project else org_url+ payload = get_json(client, f"{scope}/_apis/git/repositories")+ for repo in payload.get("value") or []:+ if not repo.get("isDisabled"):+ yield repo+++def default_branch_of(repo: dict[str, Any]) -> str:+ """Return the repository default branch name without the ``refs/heads/`` prefix."""+ return (repo.get("defaultBranch") or "refs/heads/main").removeprefix("refs/heads/")+++def should_skip_path(path: str) -> bool:+ """Skip build output, vendored code, VCS metadata and binary assets."""+ lowered = path.lower()+ if any(segment in lowered for segment in EXCLUDED_PATH_SEGMENTS):+ return True+ if lowered.rsplit("/", 1)[-1] in SKIPPED_FILENAMES:+ return True+ return get_file_ext(lowered) in BINARY_EXTENSIONS+++def document_extension(path: str) -> str:+ """Resolve the extension RAGFlow parses the file with.++ Files such as ``Dockerfile``, ``Makefile`` and ``LICENSE`` carry no+ extension; they are plain text, and an empty extension would leave the+ downstream parser without a handler.+ """+ return get_file_ext(path) or ".txt"+++def list_items(client: httpx.Client, repo_api_url: str, branch: str) -> list[dict[str, Any]]:+ """List every file in a repository at ``branch``.++ ``latestProcessedChange=true`` makes Azure DevOps return the last commit for+ each item, which supplies ``doc_updated_at`` without one extra request per+ file.+ """+ payload = get_json(+ client,+ f"{repo_api_url}/items",+ {+ "recursionLevel": "Full",+ "includeContentMetadata": "true",+ "latestProcessedChange": "true",+ "versionDescriptor.versionType": "branch",+ "versionDescriptor.version": branch,+ },+ )+ return [item for item in (payload.get("value") or []) if item.get("gitObjectType") == "blob" and not item.get("isFolder") and not should_skip_path(item.get("path", ""))]+++@retry_builder(+ tries=6,+ delay=1,+ backoff=2,+ max_delay=30,+ exceptions=(AzureDevOpsRetriableError, httpx.RequestError),+)+@rate_limit_builder(max_calls=120, period=60)+def azure_devops_get_bytes(+ client: httpx.Client,+ url: str,+ params: dict[str, Any],+ max_bytes: int,+) -> bytes | None:+ """Stream a raw response, stopping once it exceeds ``max_bytes``.++ Reading the whole body first and checking the size afterwards would let a+ single oversized repository file allocate unbounded memory, so the limit is+ enforced while the body is still being read.+ """+ with client.stream("GET", url, params=params, timeout=REQUEST_TIMEOUT_SECONDS) as response:+ raise_for_auth(response, expect_json=False)++ status = response.status_code+ if status == 429:+ sleep_for_retry_after(response.headers.get("Retry-After"))+ raise AzureDevOpsRetriableError("Azure DevOps rate limit exceeded (429).")+ if 500 <= status < 600:+ raise AzureDevOpsRetriableError(f"Azure DevOps server error: {status}")+ if 400 <= status < 500:+ raise AzureDevOpsNonRetriableError(f"Azure DevOps client error: {status} for {url}")++ chunks: list[bytes] = []+ size = 0+ for chunk in response.iter_bytes():+ size += len(chunk)+ if size > max_bytes:+ logging.warning(+ "[AzureDevOps] skipping %s: larger than the %d byte limit",+ params.get("path") or url,+ max_bytes,+ )+ return None+ chunks.append(chunk)++ return b"".join(chunks)+++def fetch_file_content(client: httpx.Client, repo_api_url: str, path: str, branch: str) -> bytes | None:+ """Download a single file, or return ``None`` when it is too large to index."""+ return azure_devops_get_bytes(+ client,+ f"{repo_api_url}/items",+ {+ "api-version": API_VERSION,+ "path": path,+ "includeContent": "true",+ "$format": "text",+ "versionDescriptor.versionType": "branch",+ "versionDescriptor.version": branch,+ },+ MAX_FILE_BYTES,+ )+++def pull_request_may_be_truncated(pull_request: dict[str, Any]) -> bool:+ """Report whether a listed pull request needs a detail fetch.++ Descriptions shorter than the limit came back whole, so the extra request is+ only paid for the few pull requests that could have been cut off.+ """+ description = pull_request.get("description") or ""+ return len(description) >= PULL_REQUEST_DESCRIPTION_LIMIT+++def fetch_pull_request(client: httpx.Client, repo_api_url: str, pull_request_id: Any) -> dict[str, Any]:+ """Fetch one pull request with its untruncated description."""+ return get_json(client, f"{repo_api_url}/pullrequests/{pull_request_id}")+++def _parse_timestamp(value: Any) -> datetime | None:+ """Parse an Azure DevOps ISO-8601 timestamp into an aware UTC datetime."""+ if not isinstance(value, str):+ return None+ try:+ return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)+ except ValueError:+ return None+++def code_document_id(organization: str, project: str, repo: str, path: str) -> str:+ """Build the stable document id for a source file."""+ return f"{DocumentSource.AZURE_DEVOPS.value}:{organization}:{project}:{repo}:file:{path}"+++def pull_request_document_id(organization: str, project: str, repo: str, pr_id: Any) -> str:+ """Build the stable document id for a pull request."""+ return f"{DocumentSource.AZURE_DEVOPS.value}:{organization}:{project}:{repo}:pr:{pr_id}"+++def map_item_to_document(+ item: dict[str, Any],+ content: bytes,+ organization: str,+ org_url: str,+ project: str,+ repo_name: str,+ branch: str,+) -> Document:+ """Map a repository file to a Document."""+ path = item.get("path", "").lstrip("/")+ change = item.get("latestProcessedChange") or {}+ committer = change.get("committer") or change.get("author") or {}+ updated_at = _parse_timestamp(committer.get("date"))+ web_url = f"{org_url}/{quote(project, safe='')}/_git/{quote(repo_name, safe='')}?path=/{path}&version=GB{branch}"++ return Document(+ id=code_document_id(organization, project, repo_name, path),+ blob=content,+ source=DocumentSource.AZURE_DEVOPS,+ semantic_identifier=path.rsplit("/", 1)[-1] or path,+ extension=document_extension(path),+ doc_updated_at=updated_at or datetime.now(tz=timezone.utc),+ size_bytes=len(content),+ primary_owners=[BasicExpertInfo(display_name=committer["name"])] if committer.get("name") else [],+ metadata={+ "type": "CodeFile",+ "path": path,+ "ref": branch,+ "project": project,+ "repository": repo_name,+ "commit_id": change.get("commitId") or "",+ "web_url": web_url,+ },+ )+++def map_pull_request_to_document(+ pull_request: dict[str, Any],+ organization: str,+ org_url: str,+ project: str,+ repo_name: str,+) -> Document:+ """Map a pull request, with its description and review metadata, to a Document."""+ pr_id = pull_request.get("pullRequestId")+ title = pull_request.get("title") or f"PR {pr_id}"+ description = pull_request.get("description") or ""+ status = pull_request.get("status") or "unknown"+ created_by = (pull_request.get("createdBy") or {}).get("displayName") or ""+ reviewers = [r.get("displayName", "") for r in pull_request.get("reviewers") or []]+ source_branch = (pull_request.get("sourceRefName") or "").removeprefix("refs/heads/")+ target_branch = (pull_request.get("targetRefName") or "").removeprefix("refs/heads/")+ created_at = _parse_timestamp(pull_request.get("creationDate"))+ closed_at = _parse_timestamp(pull_request.get("closedDate"))+ web_url = f"{org_url}/{quote(project, safe='')}/_git/{quote(repo_name, safe='')}/pullrequest/{pr_id}"++ text = (+ f"Pull Request Information:\n"+ f"- Pull Request ID: {pr_id}\n"+ f"- Title: {title}\n"+ f"- Status: {status}\n"+ f"- Repository: {project}/{repo_name}\n"+ f"- Source Branch: {source_branch}\n"+ f"- Target Branch: {target_branch}\n"+ f"- Created By: {created_by}\n"+ f"- Reviewers: {', '.join(name for name in reviewers if name) or 'N/A'}\n"+ f"- Created On: {created_at.date().isoformat() if created_at else 'N/A'}\n"+ f"- Closed On: {closed_at.date().isoformat() if closed_at else 'N/A'}\n"+ f"\nDescription:\n{description}\n"+ )+ blob = text.encode("utf-8")++ return Document(+ id=pull_request_document_id(organization, project, repo_name, pr_id),+ blob=blob,+ source=DocumentSource.AZURE_DEVOPS,+ semantic_identifier=f"PR #{pr_id}: {title}",+ extension=".txt",+ doc_updated_at=closed_at or created_at or datetime.now(tz=timezone.utc),+ size_bytes=len(blob),+ primary_owners=[BasicExpertInfo(display_name=created_by)] if created_by else [],+ metadata={+ "type": "PullRequest",+ "pull_request_id": str(pr_id),+ "status": status,+ "project": project,+ "repository": repo_name,+ "source_branch": source_branch,+ "target_branch": target_branch,+ "web_url": web_url,+ },+ )+++def pull_request_updated_at(pull_request: dict[str, Any]) -> datetime | None:+ """Best available "last activity" timestamp for a pull request."""+ return _parse_timestamp(pull_request.get("closedDate")) or _parse_timestamp(pull_request.get("creationDate"))+++def pull_request_in_window(+ pull_request: dict[str, Any],+ window_start: datetime,+ window_end: datetime,+) -> bool:+ """Decide whether a pull request belongs to the polling window.++ Azure DevOps exposes no dependable "last updated" timestamp for pull+ requests. ``closedDate`` is reliable, so completed and abandoned ones are+ filtered on it. An active pull request can have its description, reviewers+ or branches changed long after it was created, and filtering those on+ ``creationDate`` would leave the indexed document stale — so they are always+ re-indexed.+ """+ if (pull_request.get("status") or "").lower() not in ("completed", "abandoned"):+ return True++ closed_at = _parse_timestamp(pull_request.get("closedDate"))+ if closed_at is None:+ return True+ return window_start <= closed_at <= window_enddiff --git a/common/data_source/config.py b/common/data_source/config.pyindex 0a47d1a402..3561b621fd 100644--- a/common/data_source/config.py+++ b/common/data_source/config.py@@ -77,6 +77,7 @@ class DocumentSource(str, Enum):OUTLOOK = "outlook"SALESFORCE = "salesforce"AZURE_BLOB = "azure_blob"+ AZURE_DEVOPS = "azure_devops"class FileOrigin(str, Enum):diff --git a/docs/guides/data_source/data_source_categories_and_selection.md b/docs/guides/data_source/data_source_categories_and_selection.mdindex dbdeba79a0..9245c4806d 100644--- a/docs/guides/data_source/data_source_categories_and_selection.md+++ b/docs/guides/data_source/data_source_categories_and_selection.md@@ -18,7 +18,7 @@ RAGFlow supports many data sources. It is recommended to first select a major ca**Databases and data warehouses**: Used to synchronize structured records or query results. This category is suitable for business data Q&A, metric knowledge retention, and retrieval of data analysis results. It includes MySQL, PostgreSQL, and BigQuery.-**Code repositories and project management**: Used to synchronize code collaboration, issues, pull requests, tasks, and project records. This category is suitable for technical knowledge retention and project history queries. It includes GitHub, GitLab, Bitbucket, Jira, and Asana.+**Code repositories and project management**: Used to synchronize code collaboration, issues, pull requests, tasks, and project records. This category is suitable for technical knowledge retention and project history queries. It includes GitHub, GitLab, Bitbucket, Azure DevOps, Jira, and Asana.**Email, messages, and collaborative communication**: Used to synchronize emails, channel messages, and team discussion content. This category is suitable for customer service communication, business correspondence, and collaborative record queries. It includes Gmail, Outlook, IMAP, Microsoft Teams, Slack, and Discord.diff --git a/docs/guides/data_source/data_source_configuration.md b/docs/guides/data_source/data_source_configuration.mdindex e1d57b5e8d..a22487acf7 100644--- a/docs/guides/data_source/data_source_configuration.md+++ b/docs/guides/data_source/data_source_configuration.md@@ -390,6 +390,33 @@ The Bitbucket data source is used to synchronize Bitbucket repository content, i+## Azure DevOps++The Azure DevOps data source is used to synchronize Azure Repos source files and pull requests to a RAGFlow knowledge base. After configuration, repository content and code review history can be queried.++**Permission requirements**: The personal access token (PAT) must have the **Code (Read)** scope for the target organization.++**Account version requirements**: Both Azure DevOps Services (`dev.azure.com`) and self-hosted Azure DevOps Server are supported. For a self-hosted server, provide the full collection URL as the organization, for example `https://tfs.contoso.com/DefaultCollection`. The URL must use HTTPS, because the personal access token is sent in the `Authorization` header.++**Configuration parameters**:++- **Name**: Customize the name in RAGFlow to identify this Azure DevOps connection.+- **Azure DevOps Personal Access Token**: The PAT used to access the organization.+- **Azure DevOps Organization**: The organization name, or the collection URL of a self-hosted server.+- **Index Mode**: Choose whether to index every repository in the organization, only selected team projects, or only selected repositories.+- **Projects**: Comma separated team project names, used when indexing by project.+- **Repositories**: Comma separated repositories, used when indexing by repository. Use `project/repo` to disambiguate repositories that share a name across projects.+- **Content Types**: Choose whether to index source files, pull requests, or both.++Pull request descriptions are re-fetched individually when they reach the 400+character limit the list endpoint truncates at, so long descriptions are indexed+in full. Completed and abandoned pull requests are filtered by their close date;+active ones are always re-indexed, because Azure DevOps exposes no dependable+"last updated" timestamp for them.+- **Sync deleted files**: After this is enabled, content deleted from the external system is removed from the knowledge base index.++Build output and vendored directories such as `node_modules`, `bin`, `obj`, `dist`, and `vendor` are skipped, along with binary files and files larger than 1 MB.+## JiraThe Jira data source is used to synchronize issues, comments, and project records in Jira to a RAGFlow knowledge base. After configuration, project tasks, requirements, bugs, and handling records can be queried.diff --git a/internal/syncer/connector/azure_devops.go b/internal/syncer/connector/azure_devops.gonew file mode 100644index 0000000000..b5944ea6d1--- /dev/null+++ b/internal/syncer/connector/azure_devops.go@@ -0,0 +1,1204 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "path"+ "sort"+ "strconv"+ "strings"+ "time"+ "unicode/utf8"+)++const (+ azureDevOpsAPIVersion = "7.1"+ azureDevOpsHostedBaseURL = "https://dev.azure.com"+ azureDevOpsPRPageSize = 100+ // The pull request list endpoint truncates descriptions at 400 characters;+ // only the single pull request endpoint returns the full text.+ azureDevOpsPRDescriptionLimit = 400+ azureDevOpsMaxFileBytes = 1_000_000+ defaultAzureDevOpsBatchSize = 50++ azureDevOpsIndexModeOrganization = "organization"+ azureDevOpsIndexModeProjects = "projects"+ azureDevOpsIndexModeRepositories = "repositories"++ azureDevOpsContentCode = "code"+ azureDevOpsContentPullRequests = "pull_requests"+ azureDevOpsContentBoth = "both"++ azureDevOpsStageCode = "code"+ azureDevOpsStagePullRequests = "pull_requests"+)++var (+ azureDevOpsRetryTries = 6+ azureDevOpsRetryBaseDelay = 1 * time.Second+ azureDevOpsRetryBackoff = 2.0+ azureDevOpsRetryMaxDelay = 30 * time.Second++ // Build output and vendored dependencies carry no retrievable signal.+ azureDevOpsExcludedSegments = []string{+ "/node_modules/", "/bin/", "/obj/", "/dist/", "/build/", "/target/",+ "/vendor/", "/packages/", "/.git/", "/__pycache__/", "/.venv/",+ }++ azureDevOpsBinaryExtensions = map[string]struct{}{+ ".png": {}, ".jpg": {}, ".jpeg": {}, ".gif": {}, ".bmp": {}, ".ico": {},+ ".svg": {}, ".webp": {}, ".pdf": {}, ".zip": {}, ".gz": {}, ".tar": {},+ ".7z": {}, ".rar": {}, ".jar": {}, ".war": {}, ".dll": {}, ".exe": {},+ ".so": {}, ".dylib": {}, ".pdb": {}, ".class": {}, ".pyc": {},+ ".woff": {}, ".woff2": {}, ".ttf": {}, ".eot": {}, ".otf": {},+ ".mp3": {}, ".mp4": {}, ".avi": {}, ".mov": {}, ".psd": {},+ ".xlsx": {}, ".docx": {},+ }++ // Version-control metadata exists in every repository and adds only noise.+ azureDevOpsSkippedFilenames = map[string]struct{}{+ ".gitattributes": {}, ".gitignore": {}, ".gitkeep": {},+ ".gitmodules": {}, ".dockerignore": {}, ".editorconfig": {},+ }+)++// azureDevOpsHTTPError carries the status of a failed Azure DevOps call.+type azureDevOpsHTTPError struct {+ Status int+ Body string+}++func (e *azureDevOpsHTTPError) Error() string {+ return fmt.Sprintf("azure devops request failed with status %d: %s", e.Status, e.Body)+}++// AzureDevOpsConnector reads Azure Repos source files and pull requests.+type AzureDevOpsConnector struct {+ organization string+ indexMode string+ projects []string+ repositories []string+ contentTypes string+ pat string+ batchSize int+ baseURL string+ httpClient *http.Client+}++// azureDevOpsRepository is one repository selected for indexing.+type azureDevOpsRepository struct {+ Project string `json:"project"`+ Name string `json:"name"`+ Branch string `json:"branch"`+}++// Key identifies the repository inside a resume cursor.+func (r azureDevOpsRepository) Key() string {+ return r.Project + "/" + r.Name+}++type azureDevOpsChange struct {+ CommitID string `json:"commitId"`+ Committer struct {+ Name string `json:"name"`+ Date time.Time `json:"date"`+ } `json:"committer"`+ Author struct {+ Name string `json:"name"`+ Date time.Time `json:"date"`+ } `json:"author"`+}++type azureDevOpsItem struct {+ Path string `json:"path"`+ GitObjectType string `json:"gitObjectType"`+ IsFolder bool `json:"isFolder"`+ LatestProcessedChange *azureDevOpsChange `json:"latestProcessedChange"`+}++type azureDevOpsPullRequest struct {+ PullRequestID int `json:"pullRequestId"`+ Title string `json:"title"`+ Description string `json:"description"`+ Status string `json:"status"`+ SourceRefName string `json:"sourceRefName"`+ TargetRefName string `json:"targetRefName"`+ CreationDate *time.Time `json:"creationDate"`+ ClosedDate *time.Time `json:"closedDate"`+ CreatedBy struct {+ DisplayName string `json:"displayName"`+ } `json:"createdBy"`+ Reviewers []struct {+ DisplayName string `json:"displayName"`+ } `json:"reviewers"`+}++// NewAzureDevOpsConnector creates an Azure DevOps connector from config.+func NewAzureDevOpsConnector(config map[string]any) (*AzureDevOpsConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ organization := strings.TrimSpace(stringConfig(config["organization"]))++ connector := &AzureDevOpsConnector{+ organization: organization,+ indexMode: firstNonEmpty(strings.TrimSpace(stringConfig(config["index_mode"])), azureDevOpsIndexModeOrganization),+ projects: splitAzureDevOpsList(stringConfig(config["projects"])),+ repositories: splitAzureDevOpsList(stringConfig(config["repositories"])),+ contentTypes: firstNonEmpty(strings.TrimSpace(stringConfig(config["content_types"])), azureDevOpsContentBoth),+ pat: strings.TrimSpace(stringConfig(credentials["azure_devops_pat"])),+ batchSize: configInt(config["batch_size"], defaultAzureDevOpsBatchSize),+ baseURL: azureDevOpsOrganizationURL(organization),+ httpClient: &http.Client{Timeout: 60 * time.Second},+ }+ return connector, nil+}++// azureDevOpsOrganizationURL resolves the API root of a hosted organization or+// a self-hosted Azure DevOps Server collection.+func azureDevOpsOrganizationURL(organization string) string {+ if organization == "" {+ return ""+ }+ if strings.HasPrefix(organization, "http://") {+ // Rejected in checkSettings; never build a client that would send the+ // personal access token in cleartext.+ return ""+ }+ if strings.HasPrefix(organization, "https://") {+ return strings.TrimRight(organization, "/")+ }+ return azureDevOpsHostedBaseURL + "/" + url.PathEscape(organization)+}++func splitAzureDevOpsList(value string) []string {+ parts := strings.Split(value, ",")+ items := make([]string, 0, len(parts))+ for _, part := range parts {+ if trimmed := strings.TrimSpace(part); trimmed != "" {+ items = append(items, trimmed)+ }+ }+ return items+}++func (c *AzureDevOpsConnector) indexesCode() bool {+ return c.contentTypes == azureDevOpsContentCode || c.contentTypes == azureDevOpsContentBoth+}++func (c *AzureDevOpsConnector) indexesPullRequests() bool {+ return c.contentTypes == azureDevOpsContentPullRequests || c.contentTypes == azureDevOpsContentBoth+}++func (c *AzureDevOpsConnector) checkSettings() error {+ if c == nil {+ return fmt.Errorf("azure devops connector is nil")+ }+ if c.organization == "" {+ return fmt.Errorf("Invalid connector settings: 'organization' must be provided")+ }+ if c.pat == "" {+ return fmt.Errorf("Missing azure_devops_pat in credentials")+ }+ if strings.HasPrefix(c.organization, "http://") {+ return fmt.Errorf("Invalid connector settings: Azure DevOps collection URLs must use HTTPS, the personal access token is sent in the Authorization header")+ }+ switch c.indexMode {+ case azureDevOpsIndexModeOrganization, azureDevOpsIndexModeProjects, azureDevOpsIndexModeRepositories:+ default:+ return fmt.Errorf("Invalid connector settings: unsupported index mode %q", c.indexMode)+ }+ switch c.contentTypes {+ case azureDevOpsContentCode, azureDevOpsContentPullRequests, azureDevOpsContentBoth:+ default:+ return fmt.Errorf("Invalid connector settings: unsupported content types %q", c.contentTypes)+ }+ if c.indexMode == azureDevOpsIndexModeProjects && len(c.projects) == 0 {+ return fmt.Errorf("Invalid connector settings: at least one project is required when indexing by project")+ }+ if c.indexMode == azureDevOpsIndexModeRepositories && len(c.repositories) == 0 {+ return fmt.Errorf("Invalid connector settings: at least one repository is required when indexing by repository")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ return nil+}++// Validate validates Azure DevOps connector settings and credentials.+func (c *AzureDevOpsConnector) Validate(ctx context.Context) error {+ if err := c.checkSettings(); err != nil {+ return err+ }+ repos, err := c.listRepositories(ctx)+ if err != nil {+ return err+ }+ if len(repos) == 0 {+ return fmt.Errorf("found no repositories for Azure DevOps organization %s", c.organization)+ }+ return nil+}++// ValidateConnectorSetting validates Azure DevOps settings from an unsaved config.+func (c *AzureDevOpsConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()++ if err := c.checkSettings(); err != nil {+ return err+ }++ var payload map[string]any+ err := c.getJSON(ctx, c.apiURL("/_apis/projects", url.Values{"$top": {"1"}}), &payload)+ if err == nil {+ return nil+ }++ httpErr, ok := err.(*azureDevOpsHTTPError)+ if !ok {+ return err+ }+ switch httpErr.Status {+ case http.StatusNonAuthoritativeInfo, http.StatusUnauthorized:+ return fmt.Errorf("Invalid or expired Azure DevOps personal access token.")+ case http.StatusForbidden:+ return fmt.Errorf("Personal access token lacks the required 'Code (Read)' scope (HTTP 403).")+ case http.StatusNotFound:+ return fmt.Errorf("Azure DevOps organization not found: %s", c.organization)+ }+ return err+}++// OpenSync opens one Azure DevOps sync session.+func (c *AzureDevOpsConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ repos, err := c.listRepositories(ctx)+ if err != nil {+ return nil, err+ }+ if len(repos) == 0 {+ return nil, fmt.Errorf("found no repositories for Azure DevOps organization %s", c.organization)+ }++ session := &azureDevOpsSyncSession{+ connector: c,+ repos: repos,+ batchSize: c.batchSize,+ request: request,+ stage: c.initialStage(),+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune opens one complete Azure DevOps prune snapshot session.+func (c *AzureDevOpsConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ repos, err := c.listRepositories(ctx)+ if err != nil {+ return nil, err+ }+ return &azureDevOpsPruneSession{connector: c, repos: repos, batchSize: c.batchSize, stage: c.initialStage()}, nil+}++func (c *AzureDevOpsConnector) initialStage() string {+ if c.indexesCode() {+ return azureDevOpsStageCode+ }+ return azureDevOpsStagePullRequests+}++// apiURL builds an absolute API URL with the api-version query parameter set.+func (c *AzureDevOpsConnector) apiURL(apiPath string, query url.Values) string {+ if query == nil {+ query = url.Values{}+ }+ query.Set("api-version", azureDevOpsAPIVersion)+ return c.baseURL + apiPath + "?" + query.Encode()+}++// repoAPIURL builds the git API root of one repository.+func (c *AzureDevOpsConnector) repoAPIURL(repo azureDevOpsRepository) string {+ return "/" + url.PathEscape(repo.Project) + "/_apis/git/repositories/" + url.PathEscape(repo.Name)+}++// get performs an authenticated GET, retrying throttling and server errors.+//+// Azure DevOps answers an invalid or unauthorized personal access token with+// HTTP 203 and an HTML sign-in page rather than 401, so a naive status check+// treats the sign-in page as a successful response and fails later while+// decoding JSON. That case is detected here and surfaced as an auth error.+// A maxBytes of zero reads the whole response; a positive value stops one byte+// past the limit so the caller can tell an oversized payload apart without ever+// allocating it in full.+func (c *AzureDevOpsConnector) get(ctx context.Context, apiURL string, expectJSON bool, maxBytes int64) ([]byte, error) {+ delay := azureDevOpsRetryBaseDelay+ var lastErr error++ for attempt := 0; attempt < azureDevOpsRetryTries; attempt++ {+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)+ if err != nil {+ return nil, err+ }+ request.Header.Set("Accept", "application/json")+ request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(":"+c.pat)))++ response, err := c.httpClient.Do(request)+ if err != nil {+ lastErr = err+ if sleepErr := sleepForAzureDevOps(ctx, delay); sleepErr != nil {+ return nil, sleepErr+ }+ delay = nextAzureDevOpsDelay(delay)+ continue+ }++ var reader io.Reader = response.Body+ if maxBytes > 0 {+ reader = io.LimitReader(response.Body, maxBytes+1)+ }+ body, readErr := io.ReadAll(reader)+ _ = response.Body.Close()+ if readErr != nil {+ return nil, readErr+ }++ contentType := response.Header.Get("Content-Type")+ // A repository can legitimately contain .html files, so the sign-in page+ // heuristic only applies to endpoints that return JSON.+ if response.StatusCode == http.StatusNonAuthoritativeInfo || (expectJSON && strings.Contains(contentType, "text/html")) {+ return nil, &azureDevOpsHTTPError{Status: http.StatusNonAuthoritativeInfo, Body: "sign-in page returned; the personal access token is invalid or unauthorized"}+ }++ switch {+ case response.StatusCode == http.StatusTooManyRequests:+ lastErr = &azureDevOpsHTTPError{Status: response.StatusCode, Body: "rate limit exceeded"}+ wait := azureDevOpsRetryAfter(response, delay)+ if sleepErr := sleepForAzureDevOps(ctx, wait); sleepErr != nil {+ return nil, sleepErr+ }+ delay = nextAzureDevOpsDelay(delay)+ case response.StatusCode >= 500:+ lastErr = &azureDevOpsHTTPError{Status: response.StatusCode, Body: truncateAzureDevOpsBody(body)}+ if sleepErr := sleepForAzureDevOps(ctx, delay); sleepErr != nil {+ return nil, sleepErr+ }+ delay = nextAzureDevOpsDelay(delay)+ case response.StatusCode >= 400:+ return nil, &azureDevOpsHTTPError{Status: response.StatusCode, Body: truncateAzureDevOpsBody(body)}+ default:+ return body, nil+ }+ }++ if lastErr == nil {+ lastErr = fmt.Errorf("azure devops request failed after %d attempts", azureDevOpsRetryTries)+ }+ return nil, lastErr+}++func (c *AzureDevOpsConnector) getJSON(ctx context.Context, apiURL string, out any) error {+ body, err := c.get(ctx, apiURL, true, 0)+ if err != nil {+ return err+ }+ if out == nil {+ return nil+ }+ return json.Unmarshal(body, out)+}++// azureDevOpsRetryAfter honours a Retry-After header, clamped to the backoff+// ceiling. Azure DevOps can ask for a long pause, and waiting it out verbatim+// would park a sync worker for hours.+func azureDevOpsRetryAfter(response *http.Response, fallback time.Duration) time.Duration {+ if value := response.Header.Get("Retry-After"); value != "" {+ if seconds, err := strconv.Atoi(value); err == nil && seconds > 0 {+ // Clamp before the multiplication: a large enough value overflows+ // int64, and the negative result would slip past the cap and make+ // the timer fire immediately.+ if seconds >= int(azureDevOpsRetryMaxDelay/time.Second) {+ return azureDevOpsRetryMaxDelay+ }+ return time.Duration(seconds) * time.Second+ }+ }+ return fallback+}++func nextAzureDevOpsDelay(delay time.Duration) time.Duration {+ next := time.Duration(float64(delay) * azureDevOpsRetryBackoff)+ if next > azureDevOpsRetryMaxDelay {+ return azureDevOpsRetryMaxDelay+ }+ return next+}++func sleepForAzureDevOps(ctx context.Context, delay time.Duration) error {+ timer := time.NewTimer(delay)+ defer timer.Stop()+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-timer.C:+ return nil+ }+}++func truncateAzureDevOpsBody(body []byte) string {+ const limit = 300+ if len(body) > limit {+ return string(body[:limit])+ }+ return string(body)+}++// listRepositories resolves the repositories to index, deterministically ordered.+func (c *AzureDevOpsConnector) listRepositories(ctx context.Context) ([]azureDevOpsRepository, error) {+ scopes := c.repositoryScopes()+ seen := map[string]struct{}{}+ repos := make([]azureDevOpsRepository, 0, 16)++ for _, scope := range scopes {+ apiPath := "/_apis/git/repositories"+ if scope != "" {+ apiPath = "/" + url.PathEscape(scope) + "/_apis/git/repositories"+ }++ var payload struct {+ Value []struct {+ Name string `json:"name"`+ DefaultBranch string `json:"defaultBranch"`+ IsDisabled bool `json:"isDisabled"`+ Project struct {+ Name string `json:"name"`+ } `json:"project"`+ } `json:"value"`+ }+ if err := c.getJSON(ctx, c.apiURL(apiPath, nil), &payload); err != nil {+ return nil, err+ }++ for _, item := range payload.Value {+ if item.IsDisabled || item.Name == "" {+ continue+ }+ project := item.Project.Name+ if project == "" {+ project = scope+ }+ if project == "" || !c.matchesRepositoryFilter(project, item.Name) {+ continue+ }+ repo := azureDevOpsRepository{+ Project: project,+ Name: item.Name,+ Branch: strings.TrimPrefix(firstNonEmpty(item.DefaultBranch, "refs/heads/main"), "refs/heads/"),+ }+ if _, exists := seen[repo.Key()]; exists {+ continue+ }+ seen[repo.Key()] = struct{}{}+ repos = append(repos, repo)+ }+ }++ sort.Slice(repos, func(i, j int) bool { return repos[i].Key() < repos[j].Key() })+ return repos, nil+}++// repositoryScopes returns the projects to query, or a single empty scope to+// use the organization-wide endpoint that returns every repository at once.+func (c *AzureDevOpsConnector) repositoryScopes() []string {+ switch c.indexMode {+ case azureDevOpsIndexModeProjects:+ if len(c.projects) > 0 {+ return c.projects+ }+ case azureDevOpsIndexModeRepositories:+ qualified := map[string]struct{}{}+ for _, entry := range c.repositories {+ if project, _, found := strings.Cut(entry, "/"); found && project != "" {+ qualified[project] = struct{}{}+ }+ }+ if len(qualified) > 0 {+ scopes := make([]string, 0, len(qualified))+ for project := range qualified {+ scopes = append(scopes, project)+ }+ sort.Strings(scopes)+ return scopes+ }+ }+ return []string{""}+}++// matchesRepositoryFilter accepts "project/repo" and bare repository names.+//+// Azure DevOps repository names are unique per project rather than per+// organization, so the qualified form is the unambiguous one.+func (c *AzureDevOpsConnector) matchesRepositoryFilter(project, name string) bool {+ if c.indexMode != azureDevOpsIndexModeRepositories || len(c.repositories) == 0 {+ return true+ }+ qualified := project + "/" + name+ for _, entry := range c.repositories {+ if entry == name || entry == qualified {+ return true+ }+ }+ return false+}++// listItems lists every indexable file of a repository at its default branch.+//+// latestProcessedChange returns the last commit of each item in the same+// response, which supplies both the update timestamp and the fingerprint+// without one extra request per file.+func (c *AzureDevOpsConnector) listItems(ctx context.Context, repo azureDevOpsRepository) ([]azureDevOpsItem, error) {+ query := url.Values{+ "recursionLevel": {"Full"},+ "includeContentMetadata": {"true"},+ "latestProcessedChange": {"true"},+ "versionDescriptor.versionType": {"branch"},+ "versionDescriptor.version": {repo.Branch},+ }++ var payload struct {+ Value []azureDevOpsItem `json:"value"`+ }+ if err := c.getJSON(ctx, c.apiURL(c.repoAPIURL(repo)+"/items", query), &payload); err != nil {+ return nil, err+ }++ items := make([]azureDevOpsItem, 0, len(payload.Value))+ for _, item := range payload.Value {+ if item.GitObjectType != "blob" || item.IsFolder || shouldSkipAzureDevOpsPath(item.Path) {+ continue+ }+ items = append(items, item)+ }+ return items, nil+}++// fetchFile downloads one file, returning nil when it is too large to index.+func (c *AzureDevOpsConnector) fetchFile(ctx context.Context, repo azureDevOpsRepository, filePath string) ([]byte, error) {+ query := url.Values{+ "path": {filePath},+ "includeContent": {"true"},+ "$format": {"text"},+ "versionDescriptor.versionType": {"branch"},+ "versionDescriptor.version": {repo.Branch},+ }+ body, err := c.get(ctx, c.apiURL(c.repoAPIURL(repo)+"/items", query), false, azureDevOpsMaxFileBytes)+ if err != nil {+ return nil, err+ }+ if int64(len(body)) > azureDevOpsMaxFileBytes {+ return nil, nil+ }+ return body, nil+}++// listPullRequests returns one page of pull requests in every state.+func (c *AzureDevOpsConnector) listPullRequests(ctx context.Context, repo azureDevOpsRepository, skip int) ([]azureDevOpsPullRequest, error) {+ query := url.Values{+ "searchCriteria.status": {"all"},+ "$top": {strconv.Itoa(azureDevOpsPRPageSize)},+ "$skip": {strconv.Itoa(skip)},+ }+ var payload struct {+ Value []azureDevOpsPullRequest `json:"value"`+ }+ if err := c.getJSON(ctx, c.apiURL(c.repoAPIURL(repo)+"/pullrequests", query), &payload); err != nil {+ return nil, err+ }+ return payload.Value, nil+}++// fetchPullRequest fetches one pull request with its untruncated description.+func (c *AzureDevOpsConnector) fetchPullRequest(ctx context.Context, repo azureDevOpsRepository, pullRequestID int) (azureDevOpsPullRequest, error) {+ var pullRequest azureDevOpsPullRequest+ apiPath := fmt.Sprintf("%s/pullrequests/%d", c.repoAPIURL(repo), pullRequestID)+ if err := c.getJSON(ctx, c.apiURL(apiPath, nil), &pullRequest); err != nil {+ return azureDevOpsPullRequest{}, err+ }+ return pullRequest, nil+}++// azureDevOpsPullRequestMayBeTruncated reports whether a listed pull request+// needs a detail fetch.+//+// Descriptions shorter than the limit came back whole, so the extra request is+// only paid for the few pull requests that could have been cut off.+func azureDevOpsPullRequestMayBeTruncated(pullRequest azureDevOpsPullRequest) bool {+ return utf8.RuneCountInString(pullRequest.Description) >= azureDevOpsPRDescriptionLimit+}++// shouldSkipAzureDevOpsPath drops build output, vendored code, version-control+// metadata and binary assets.+func shouldSkipAzureDevOpsPath(itemPath string) bool {+ lowered := strings.ToLower(itemPath)+ for _, segment := range azureDevOpsExcludedSegments {+ if strings.Contains(lowered, segment) {+ return true+ }+ }+ if _, skipped := azureDevOpsSkippedFilenames[path.Base(lowered)]; skipped {+ return true+ }+ _, binary := azureDevOpsBinaryExtensions[path.Ext(lowered)]+ return binary+}++// azureDevOpsDocumentExtension resolves the extension the file is parsed with.+//+// Files such as Dockerfile, Makefile and LICENSE carry no extension, and an+// empty extension would leave the downstream parser without a handler.+func azureDevOpsDocumentExtension(itemPath string) string {+ if extension := strings.ToLower(path.Ext(itemPath)); extension != "" {+ return extension+ }+ return ".txt"+}++func azureDevOpsCodeSourceID(organization string, repo azureDevOpsRepository, filePath string) string {+ return fmt.Sprintf("azure_devops:%s:%s:%s:file:%s", organization, repo.Project, repo.Name, strings.TrimPrefix(filePath, "/"))+}++func azureDevOpsPullRequestSourceID(organization string, repo azureDevOpsRepository, pullRequestID int) string {+ return fmt.Sprintf("azure_devops:%s:%s:%s:pr:%d", organization, repo.Project, repo.Name, pullRequestID)+}++// includeAzureDevOpsItem decides whether a file needs to be re-synced.+//+// The commit id of the last change acts as the fingerprint, so unchanged files+// are skipped before their content is downloaded.+func includeAzureDevOpsItem(request SyncRequest, sourceID string, item azureDevOpsItem) bool {+ if request.FromBeginning {+ return true+ }+ fingerprint := azureDevOpsItemFingerprint(item)+ if len(request.Fingerprints) > 0 {+ stored, ok := request.Fingerprints[sourceID]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }+ updatedAt := azureDevOpsItemUpdatedAt(item)+ if updatedAt.IsZero() {+ return true+ }+ return !beforeOrAtWindowStart(updatedAt, request.WindowStart) && !afterWindowEnd(updatedAt, request.WindowEnd)+}++func azureDevOpsItemFingerprint(item azureDevOpsItem) string {+ if item.LatestProcessedChange == nil {+ return ""+ }+ return item.LatestProcessedChange.CommitID+}++func azureDevOpsItemUpdatedAt(item azureDevOpsItem) time.Time {+ if item.LatestProcessedChange == nil {+ return time.Time{}+ }+ if !item.LatestProcessedChange.Committer.Date.IsZero() {+ return item.LatestProcessedChange.Committer.Date+ }+ return item.LatestProcessedChange.Author.Date+}++func azureDevOpsPullRequestUpdatedAt(pullRequest azureDevOpsPullRequest) time.Time {+ if pullRequest.ClosedDate != nil && !pullRequest.ClosedDate.IsZero() {+ return *pullRequest.ClosedDate+ }+ if pullRequest.CreationDate != nil {+ return *pullRequest.CreationDate+ }+ return time.Time{}+}++// includeAzureDevOpsPullRequest applies the sync window client-side; the Azure+// DevOps pull request endpoint has no dependable "updated since" filter.+func includeAzureDevOpsPullRequest(request SyncRequest, sourceID string, pullRequest azureDevOpsPullRequest) bool {+ if request.FromBeginning {+ return true+ }+ if len(request.Fingerprints) > 0 {+ fingerprint := azureDevOpsPullRequestFingerprint(pullRequest)+ stored, ok := request.Fingerprints[sourceID]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }++ // Azure DevOps exposes no dependable "last updated" timestamp for pull+ // requests. closedDate is reliable, so completed and abandoned ones are+ // filtered on it; an active pull request can change at any time, and+ // filtering it on creationDate would leave the indexed document stale.+ status := strings.ToLower(pullRequest.Status)+ if status != "completed" && status != "abandoned" {+ return true+ }+ updatedAt := azureDevOpsPullRequestUpdatedAt(pullRequest)+ if updatedAt.IsZero() {+ return true+ }+ return !beforeOrAtWindowStart(updatedAt, request.WindowStart) && !afterWindowEnd(updatedAt, request.WindowEnd)+}++func azureDevOpsPullRequestFingerprint(pullRequest azureDevOpsPullRequest) string {+ updatedAt := azureDevOpsPullRequestUpdatedAt(pullRequest)+ if updatedAt.IsZero() {+ return ""+ }+ return pullRequest.Status + ":" + updatedAt.UTC().Format(time.RFC3339Nano)+}++// buildAzureDevOpsCodeDocument maps a repository file to a source document.+func (c *AzureDevOpsConnector) buildAzureDevOpsCodeDocument(repo azureDevOpsRepository, item azureDevOpsItem, content []byte) SourceDocument {+ relativePath := strings.TrimPrefix(item.Path, "/")+ updatedAt := azureDevOpsItemUpdatedAt(item)+ if updatedAt.IsZero() {+ updatedAt = time.Now().UTC()+ }++ webURL := fmt.Sprintf("%s/%s/_git/%s?path=/%s&version=GB%s",+ c.baseURL, url.PathEscape(repo.Project), url.PathEscape(repo.Name), relativePath, repo.Branch)++ commitID := azureDevOpsItemFingerprint(item)+ committer := ""+ if item.LatestProcessedChange != nil {+ committer = firstNonEmpty(item.LatestProcessedChange.Committer.Name, item.LatestProcessedChange.Author.Name)+ }++ return SourceDocument{+ SourceID: azureDevOpsCodeSourceID(c.organization, repo, relativePath),+ SemanticIdentifier: path.Base(relativePath),+ Extension: azureDevOpsDocumentExtension(relativePath),+ Blob: content,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(content)),+ Fingerprint: commitID,+ Metadata: map[string]any{+ "type": "CodeFile",+ "path": relativePath,+ "ref": repo.Branch,+ "project": repo.Project,+ "repository": repo.Name,+ "commit_id": commitID,+ "committer": committer,+ "web_url": webURL,+ },+ }+}++// buildAzureDevOpsPullRequestDocument maps a pull request to a source document.+func (c *AzureDevOpsConnector) buildAzureDevOpsPullRequestDocument(repo azureDevOpsRepository, pullRequest azureDevOpsPullRequest) SourceDocument {+ sourceBranch := strings.TrimPrefix(pullRequest.SourceRefName, "refs/heads/")+ targetBranch := strings.TrimPrefix(pullRequest.TargetRefName, "refs/heads/")++ reviewers := make([]string, 0, len(pullRequest.Reviewers))+ for _, reviewer := range pullRequest.Reviewers {+ if reviewer.DisplayName != "" {+ reviewers = append(reviewers, reviewer.DisplayName)+ }+ }++ createdOn := "N/A"+ if pullRequest.CreationDate != nil {+ createdOn = pullRequest.CreationDate.UTC().Format("2006-01-02")+ }+ closedOn := "N/A"+ if pullRequest.ClosedDate != nil {+ closedOn = pullRequest.ClosedDate.UTC().Format("2006-01-02")+ }++ var builder strings.Builder+ builder.WriteString("Pull Request Information:\n")+ fmt.Fprintf(&builder, "- Pull Request ID: %d\n", pullRequest.PullRequestID)+ fmt.Fprintf(&builder, "- Title: %s\n", pullRequest.Title)+ fmt.Fprintf(&builder, "- Status: %s\n", pullRequest.Status)+ fmt.Fprintf(&builder, "- Repository: %s/%s\n", repo.Project, repo.Name)+ fmt.Fprintf(&builder, "- Source Branch: %s\n", sourceBranch)+ fmt.Fprintf(&builder, "- Target Branch: %s\n", targetBranch)+ fmt.Fprintf(&builder, "- Created By: %s\n", pullRequest.CreatedBy.DisplayName)+ fmt.Fprintf(&builder, "- Reviewers: %s\n", firstNonEmpty(strings.Join(reviewers, ", "), "N/A"))+ fmt.Fprintf(&builder, "- Created On: %s\n", createdOn)+ fmt.Fprintf(&builder, "- Closed On: %s\n", closedOn)+ fmt.Fprintf(&builder, "\nDescription:\n%s\n", pullRequest.Description)++ blob := []byte(builder.String())+ updatedAt := azureDevOpsPullRequestUpdatedAt(pullRequest)+ if updatedAt.IsZero() {+ updatedAt = time.Now().UTC()+ }++ return SourceDocument{+ SourceID: azureDevOpsPullRequestSourceID(c.organization, repo, pullRequest.PullRequestID),+ SemanticIdentifier: fmt.Sprintf("PR #%d: %s", pullRequest.PullRequestID, pullRequest.Title),+ Extension: ".txt",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Fingerprint: azureDevOpsPullRequestFingerprint(pullRequest),+ Metadata: map[string]any{+ "type": "PullRequest",+ "pull_request_id": strconv.Itoa(pullRequest.PullRequestID),+ "status": pullRequest.Status,+ "project": repo.Project,+ "repository": repo.Name,+ "source_branch": sourceBranch,+ "target_branch": targetBranch,+ "web_url": fmt.Sprintf("%s/%s/_git/%s/pullrequest/%d", c.baseURL, url.PathEscape(repo.Project), url.PathEscape(repo.Name), pullRequest.PullRequestID),+ },+ }+}++// azureDevOpsSyncCursor is the resume position persisted between batches.+// azureDevOpsSyncCursor is the resume position persisted between batches.+//+// SourceID is the anchor: offsets alone are positions in a remote listing that+// shifts whenever a file or pull request is added or removed, so resuming on an+// offset can silently skip or repeat an item. The offset is kept only as a fast+// lookup hint, and correctness is decided by the anchor.+type azureDevOpsSyncCursor struct {+ RepoKey string `json:"repo_key"`+ Stage string `json:"stage"`+ FileOffset int `json:"file_offset,omitempty"`+ PRSkip int `json:"pr_skip,omitempty"`+ SourceID string `json:"source_id,omitempty"`+}++type azureDevOpsSyncSession struct {+ connector *AzureDevOpsConnector+ repos []azureDevOpsRepository+ repoIndex int+ stage string+ fileOffset int+ prSkip int+ items []azureDevOpsItem+ itemsRepo string+ batchSize int+ request SyncRequest+ lastSourceID string++ // Resume anchor, consumed by the first batch produced after a resume.+ resumeRepoKey string+ resumeStage string+ resumeSourceID string+}++// NextBatch returns the next Azure DevOps document batch.+func (s *azureDevOpsSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)++ for len(documents) < s.batchSize {+ if s.repoIndex >= len(s.repos) {+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }++ repo := s.repos[s.repoIndex]+ var (+ produced []SourceDocument+ err error+ )+ if s.stage == azureDevOpsStageCode {+ produced, err = s.nextCodeDocuments(ctx, repo, s.batchSize-len(documents))+ } else {+ produced, err = s.nextPullRequestDocuments(ctx, repo)+ }+ if err != nil {+ return SyncBatch{}, err+ }+ documents = append(documents, produced...)+ }++ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ s.lastSourceID = documents[len(documents)-1].SourceID+ return SyncBatch{Documents: documents, Checkpoint: s.checkpoint()}, nil+}++// Close closes the Azure DevOps sync session.+func (s *azureDevOpsSyncSession) Close() error {+ return nil+}++func (s *azureDevOpsSyncSession) checkpoint() *SyncCheckpoint {+ if s.repoIndex >= len(s.repos) {+ return nil+ }+ cursor := azureDevOpsSyncCursor{+ RepoKey: s.repos[s.repoIndex].Key(),+ Stage: s.stage,+ FileOffset: s.fileOffset,+ PRSkip: s.prSkip,+ SourceID: s.lastSourceID,+ }+ encoded, err := json.Marshal(cursor)+ if err != nil {+ return nil+ }+ return &SyncCheckpoint{Cursor: string(encoded), SourceID: s.lastSourceID}+}++// nextCodeDocuments emits up to limit files, advancing the stage when the+// repository file list is exhausted.+func (s *azureDevOpsSyncSession) nextCodeDocuments(ctx context.Context, repo azureDevOpsRepository, limit int) ([]SourceDocument, error) {+ if s.itemsRepo != repo.Key() {+ items, err := s.connector.listItems(ctx, repo)+ if err != nil {+ return nil, err+ }+ s.items = items+ s.itemsRepo = repo.Key()+ s.applyFileAnchor(repo)+ }++ documents := make([]SourceDocument, 0, limit)+ for s.fileOffset < len(s.items) && len(documents) < limit {+ item := s.items[s.fileOffset]+ s.fileOffset++++ sourceID := azureDevOpsCodeSourceID(s.connector.organization, repo, strings.TrimPrefix(item.Path, "/"))+ if !includeAzureDevOpsItem(s.request, sourceID, item) {+ continue+ }+ content, err := s.connector.fetchFile(ctx, repo, item.Path)+ if err != nil {+ return nil, err+ }+ if content == nil {+ continue+ }+ documents = append(documents, s.connector.buildAzureDevOpsCodeDocument(repo, item, content))+ }++ if s.fileOffset >= len(s.items) {+ s.advanceStage()+ }+ return documents, nil+}++// applyFileAnchor positions the file walk after the last committed document.+//+// The listing is re-fetched on every resume and can have shifted since the+// checkpoint was written, so the stored offset is treated as a hint: it is+// checked first, then the whole listing is searched for the anchor. Only when+// the anchor is gone — the file was deleted or renamed — does the offset stand+// on its own.+func (s *azureDevOpsSyncSession) applyFileAnchor(repo azureDevOpsRepository) {+ if s.resumeSourceID == "" || s.resumeRepoKey != repo.Key() || s.resumeStage != azureDevOpsStageCode {+ return+ }+ defer s.clearResume()++ if s.fileOffset > 0 && s.fileOffset <= len(s.items) {+ previous := s.items[s.fileOffset-1]+ if azureDevOpsCodeSourceID(s.connector.organization, repo, strings.TrimPrefix(previous.Path, "/")) == s.resumeSourceID {+ return+ }+ }++ for index, item := range s.items {+ if azureDevOpsCodeSourceID(s.connector.organization, repo, strings.TrimPrefix(item.Path, "/")) == s.resumeSourceID {+ s.fileOffset = index + 1+ return+ }+ }+}++// filterResumedPullRequests drops the pull requests already committed.+//+// $skip indexes into a listing that shifts as pull requests are opened, so the+// anchor decides where the page really resumes; the skip value only positions+// the request.+func (s *azureDevOpsSyncSession) filterResumedPullRequests(repo azureDevOpsRepository, pullRequests []azureDevOpsPullRequest) []azureDevOpsPullRequest {+ if s.resumeSourceID == "" || s.resumeRepoKey != repo.Key() || s.resumeStage != azureDevOpsStagePullRequests {+ return pullRequests+ }+ defer s.clearResume()++ for index, pullRequest := range pullRequests {+ if azureDevOpsPullRequestSourceID(s.connector.organization, repo, pullRequest.PullRequestID) == s.resumeSourceID {+ return pullRequests[index+1:]+ }+ }+ return pullRequests+}++func (s *azureDevOpsSyncSession) clearResume() {+ s.resumeRepoKey = ""+ s.resumeStage = ""+ s.resumeSourceID = ""+}++// nextPullRequestDocuments emits one pull request page, advancing to the next+// repository once the last page is consumed.+func (s *azureDevOpsSyncSession) nextPullRequestDocuments(ctx context.Context, repo azureDevOpsRepository) ([]SourceDocument, error) {+ pullRequests, err := s.connector.listPullRequests(ctx, repo, s.prSkip)+ if err != nil {+ return nil, err+ }+ pageSize := len(pullRequests)+ pullRequests = s.filterResumedPullRequests(repo, pullRequests)++ documents := make([]SourceDocument, 0, len(pullRequests))+ for _, pullRequest := range pullRequests {+ sourceID := azureDevOpsPullRequestSourceID(s.connector.organization, repo, pullRequest.PullRequestID)+ if !includeAzureDevOpsPullRequest(s.request, sourceID, pullRequest) {+ continue+ }+ if azureDevOpsPullRequestMayBeTruncated(pullRequest) {+ detailed, err := s.connector.fetchPullRequest(ctx, repo, pullRequest.PullRequestID)+ if err != nil {+ return nil, err+ }+ pullRequest = detailed+ }+ documents = append(documents, s.connector.buildAzureDevOpsPullRequestDocument(repo, pullRequest))+ }++ if pageSize < azureDevOpsPRPageSize {+ s.advanceRepo()+ } else {+ s.prSkip += azureDevOpsPRPageSize+ }+ return documents, nil+}++func (s *azureDevOpsSyncSession) advanceStage() {+ s.fileOffset = 0+ s.items = nil+ s.itemsRepo = ""+ if s.connector.indexesPullRequests() {+ s.stage = azureDevOpsStagePullRequests+ return+ }+ s.advanceRepo()+}++func (s *azureDevOpsSyncSession) advanceRepo() {+ s.repoIndex+++ s.stage = s.connector.initialStage()+ s.fileOffset = 0+ s.prSkip = 0+ s.items = nil+ s.itemsRepo = ""+}++// applyResume advances the session to the last committed position.+func (s *azureDevOpsSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }+ var cursor azureDevOpsSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil || cursor.RepoKey == "" {+ return+ }+ for index, repo := range s.repos {+ if repo.Key() != cursor.RepoKey {+ continue+ }+ s.repoIndex = index+ if cursor.Stage != "" {+ s.stage = cursor.Stage+ }+ s.fileOffset = cursor.FileOffset+ s.prSkip = cursor.PRSkip+ s.resumeRepoKey = cursor.RepoKey+ s.resumeStage = s.stage+ s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)+ return+ }+}++type azureDevOpsPruneSession struct {+ connector *AzureDevOpsConnector+ repos []azureDevOpsRepository+ repoIndex int+ stage string+ prSkip int+ batchSize int+}++// NextBatch returns the next Azure DevOps slim snapshot batch.+func (s *azureDevOpsPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ for s.repoIndex < len(s.repos) {+ repo := s.repos[s.repoIndex]++ if s.stage == azureDevOpsStageCode {+ items, err := s.connector.listItems(ctx, repo)+ if err != nil {+ return PruneBatch{}, err+ }+ documents := make([]SlimDocument, 0, len(items))+ for _, item := range items {+ documents = append(documents, SlimDocument{+ SourceID: azureDevOpsCodeSourceID(s.connector.organization, repo, strings.TrimPrefix(item.Path, "/")),+ })+ }+ if s.connector.indexesPullRequests() {+ s.stage = azureDevOpsStagePullRequests+ } else {+ s.advanceRepo()+ }+ if len(documents) > 0 {+ return PruneBatch{Documents: documents}, nil+ }+ continue+ }++ pullRequests, err := s.connector.listPullRequests(ctx, repo, s.prSkip)+ if err != nil {+ return PruneBatch{}, err+ }+ documents := make([]SlimDocument, 0, len(pullRequests))+ for _, pullRequest := range pullRequests {+ documents = append(documents, SlimDocument{+ SourceID: azureDevOpsPullRequestSourceID(s.connector.organization, repo, pullRequest.PullRequestID),+ })+ }+ if len(pullRequests) < azureDevOpsPRPageSize {+ s.advanceRepo()+ } else {+ s.prSkip += azureDevOpsPRPageSize+ }+ if len(documents) > 0 {+ return PruneBatch{Documents: documents}, nil+ }+ }+ return PruneBatch{}, io.EOF+}++// Close closes the Azure DevOps prune session.+func (s *azureDevOpsPruneSession) Close() error {+ return nil+}++func (s *azureDevOpsPruneSession) advanceRepo() {+ s.repoIndex+++ s.stage = s.connector.initialStage()+ s.prSkip = 0+}diff --git a/internal/syncer/connector/azure_devops_test.go b/internal/syncer/connector/azure_devops_test.gonew file mode 100644index 0000000000..c3cf2a72cd--- /dev/null+++ b/internal/syncer/connector/azure_devops_test.go@@ -0,0 +1,837 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "math"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "testing"+ "time"+)++// newTestAzureDevOpsConnector points a connector at a stub server.+func newTestAzureDevOpsConnector(t *testing.T, serverURL string, overrides map[string]any) *AzureDevOpsConnector {+ t.Helper()++ config := map[string]any{+ "organization": "contoso",+ "credentials": map[string]any{"azure_devops_pat": "token"},+ }+ for key, value := range overrides {+ config[key] = value+ }++ connector, err := NewAzureDevOpsConnector(config)+ if err != nil {+ t.Fatalf("NewAzureDevOpsConnector returned error: %v", err)+ }+ // httptest serves over http, which the connector refuses for real+ // configuration, so the stub endpoint is injected directly.+ connector.baseURL = serverURL+ return connector+}++func TestNewAzureDevOpsConnectorParsesConfig(t *testing.T) {+ connector, err := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ "index_mode": azureDevOpsIndexModeRepositories,+ "projects": " alpha , beta ",+ "repositories": "iddaa/Bayi-Portal, sportsbook",+ "content_types": azureDevOpsContentCode,+ "credentials": map[string]any{"azure_devops_pat": " token "},+ })+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }++ if connector.baseURL != "https://dev.azure.com/contoso" {+ t.Fatalf("unexpected base URL: %s", connector.baseURL)+ }+ if len(connector.projects) != 2 || connector.projects[0] != "alpha" {+ t.Fatalf("unexpected projects: %#v", connector.projects)+ }+ if len(connector.repositories) != 2 || connector.repositories[1] != "sportsbook" {+ t.Fatalf("unexpected repositories: %#v", connector.repositories)+ }+ if connector.pat != "token" {+ t.Fatalf("credentials were not trimmed: %q", connector.pat)+ }+ if connector.indexesPullRequests() {+ t.Fatal("content_types=code must not index pull requests")+ }+}++func TestAzureDevOpsOrganizationURLSupportsSelfHostedCollection(t *testing.T) {+ if got := azureDevOpsOrganizationURL("contoso"); got != "https://dev.azure.com/contoso" {+ t.Fatalf("unexpected hosted URL: %s", got)+ }+ if got := azureDevOpsOrganizationURL("https://tfs.contoso.com/DefaultCollection/"); got != "https://tfs.contoso.com/DefaultCollection" {+ t.Fatalf("unexpected self-hosted URL: %s", got)+ }+}++func TestAzureDevOpsDefaultsIndexModeAndContentTypes(t *testing.T) {+ connector, _ := NewAzureDevOpsConnector(map[string]any{"organization": "contoso"})+ if connector.indexMode != azureDevOpsIndexModeOrganization {+ t.Fatalf("unexpected index mode: %s", connector.indexMode)+ }+ if !connector.indexesCode() || !connector.indexesPullRequests() {+ t.Fatal("default content types must cover code and pull requests")+ }+}++// Azure DevOps answers an unauthorized personal access token with HTTP 203 and+// an HTML sign-in page instead of 401.+func TestAzureDevOpsSignInPageIsReportedAsAuthFailure(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/html; charset=utf-8")+ w.WriteHeader(http.StatusNonAuthoritativeInfo)+ _, _ = io.WriteString(w, "<html><body>Sign In</body></html>")+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ err := connector.ValidateConnectorSetting(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), "Invalid or expired") {+ t.Fatalf("expected an auth failure, got %v", err)+ }+}++func TestAzureDevOpsForbiddenReportsMissingScope(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusForbidden)+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ err := connector.ValidateConnectorSetting(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), "Code (Read)") {+ t.Fatalf("expected a scope error, got %v", err)+ }+}++func TestAzureDevOpsValidateRejectsMissingCredentials(t *testing.T) {+ connector, _ := NewAzureDevOpsConnector(map[string]any{"organization": "contoso"})+ if err := connector.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "azure_devops_pat") {+ t.Fatalf("expected a credential error, got %v", err)+ }+}++func TestAzureDevOpsValidateRejectsProjectModeWithoutProjects(t *testing.T) {+ connector, _ := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ "index_mode": azureDevOpsIndexModeProjects,+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })+ if err := connector.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "project") {+ t.Fatalf("expected a project error, got %v", err)+ }+}++func TestAzureDevOpsServerErrorIsRetriedThenSucceeds(t *testing.T) {+ attempts := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ attempts+++ if attempts == 1 {+ w.WriteHeader(http.StatusInternalServerError)+ return+ }+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ }))+ defer server.Close()++ previousDelay := azureDevOpsRetryBaseDelay+ azureDevOpsRetryBaseDelay = time.Millisecond+ defer func() { azureDevOpsRetryBaseDelay = previousDelay }()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ if _, err := connector.listRepositories(context.Background()); err != nil {+ t.Fatalf("retry did not recover: %v", err)+ }+ if attempts != 2 {+ t.Fatalf("expected exactly one retry, got %d attempts", attempts)+ }+}++func TestAzureDevOpsClientErrorIsNotRetried(t *testing.T) {+ attempts := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ attempts+++ w.WriteHeader(http.StatusBadRequest)+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ _, err := connector.listRepositories(context.Background())++ var httpErr *azureDevOpsHTTPError+ if !errors.As(err, &httpErr) || httpErr.Status != http.StatusBadRequest {+ t.Fatalf("expected a 400 http error, got %v", err)+ }+ if attempts != 1 {+ t.Fatalf("client errors must not be retried, got %d attempts", attempts)+ }+}++func TestAzureDevOpsListRepositoriesSkipsDisabledAndSorts(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "zeta", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ map[string]any{"name": "alpha", "defaultBranch": "refs/heads/develop", "project": map[string]any{"name": "iddaa"}},+ map[string]any{"name": "retired", "isDisabled": true, "project": map[string]any{"name": "iddaa"}},+ }})+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ repos, err := connector.listRepositories(context.Background())+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }++ if len(repos) != 2 {+ t.Fatalf("expected 2 repositories, got %d", len(repos))+ }+ if repos[0].Name != "alpha" || repos[0].Branch != "develop" {+ t.Fatalf("unexpected first repository: %#v", repos[0])+ }+ if repos[1].Key() != "iddaa/zeta" {+ t.Fatalf("unexpected repository key: %s", repos[1].Key())+ }+}++func TestAzureDevOpsRepositoryFilterAcceptsQualifiedNames(t *testing.T) {+ connector, _ := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ "index_mode": azureDevOpsIndexModeRepositories,+ "repositories": "iddaa/Bayi-Portal,sportsbook",+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })++ if !connector.matchesRepositoryFilter("iddaa", "Bayi-Portal") {+ t.Fatal("qualified name must match")+ }+ if !connector.matchesRepositoryFilter("other", "sportsbook") {+ t.Fatal("bare name must match in any project")+ }+ if connector.matchesRepositoryFilter("iddaa", "unrelated") {+ t.Fatal("unlisted repository must not match")+ }+}++func TestShouldSkipAzureDevOpsPath(t *testing.T) {+ cases := map[string]bool{+ "/src/Payments/RefundService.cs": false,+ "/Dockerfile": false,+ "/LICENSE": false,+ "/.gitattributes": true,+ "/.gitignore": true,+ "/node_modules/lib/index.js": true,+ "/Bayi-Portal.Api/bin/Debug/app.dll": true,+ "/docs/logo.png": true,+ }+ for itemPath, expected := range cases {+ if got := shouldSkipAzureDevOpsPath(itemPath); got != expected {+ t.Fatalf("shouldSkipAzureDevOpsPath(%q) = %v, want %v", itemPath, got, expected)+ }+ }+}++func TestAzureDevOpsDocumentExtensionFallsBackToText(t *testing.T) {+ if got := azureDevOpsDocumentExtension("Dockerfile"); got != ".txt" {+ t.Fatalf("expected .txt fallback, got %q", got)+ }+ if got := azureDevOpsDocumentExtension("src/App.CS"); got != ".cs" {+ t.Fatalf("expected lowercase extension, got %q", got)+ }+}++func TestAzureDevOpsCodeDocumentCarriesCommitAndWebURL(t *testing.T) {+ connector, _ := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })+ repo := azureDevOpsRepository{Project: "iddaa", Name: "Bayi-Portal", Branch: "master"}+ changed := time.Date(2026, 1, 13, 21, 53, 6, 0, time.UTC)++ item := azureDevOpsItem{Path: "/src/Payments/RefundService.cs", GitObjectType: "blob"}+ item.LatestProcessedChange = &azureDevOpsChange{CommitID: "abc123"}+ item.LatestProcessedChange.Committer.Name = "Ada Lovelace"+ item.LatestProcessedChange.Committer.Date = changed++ document := connector.buildAzureDevOpsCodeDocument(repo, item, []byte("public class RefundService {}"))++ if document.SourceID != "azure_devops:contoso:iddaa:Bayi-Portal:file:src/Payments/RefundService.cs" {+ t.Fatalf("unexpected source id: %s", document.SourceID)+ }+ if document.Extension != ".cs" || document.SemanticIdentifier != "RefundService.cs" {+ t.Fatalf("unexpected document identity: %#v", document)+ }+ if !document.UpdatedAt.Equal(changed) {+ t.Fatalf("unexpected updated at: %s", document.UpdatedAt)+ }+ if document.Fingerprint != "abc123" {+ t.Fatalf("commit id must act as the fingerprint, got %q", document.Fingerprint)+ }+ if webURL, _ := document.Metadata["web_url"].(string); !strings.Contains(webURL, "path=/src/Payments/RefundService.cs") {+ t.Fatalf("unexpected web url: %s", webURL)+ }+}++func TestAzureDevOpsPullRequestDocumentSummarisesReview(t *testing.T) {+ connector, _ := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })+ repo := azureDevOpsRepository{Project: "iddaa", Name: "Bayi-Portal", Branch: "master"}+ created := time.Date(2025, 11, 18, 6, 20, 7, 0, time.UTC)+ closed := time.Date(2025, 11, 20, 6, 20, 7, 0, time.UTC)++ pullRequest := azureDevOpsPullRequest{+ PullRequestID: 4225,+ Title: "BYS-1517 - PDF olusturma",+ Description: "Backend tarafina tasindi.",+ Status: "completed",+ SourceRefName: "refs/heads/feature/BYS-1517",+ TargetRefName: "refs/heads/master",+ CreationDate: &created,+ ClosedDate: &closed,+ }+ pullRequest.CreatedBy.DisplayName = "Ada Lovelace"+ pullRequest.Reviewers = append(pullRequest.Reviewers, struct {+ DisplayName string `json:"displayName"`+ }{DisplayName: "Grace Hopper"})++ document := connector.buildAzureDevOpsPullRequestDocument(repo, pullRequest)++ if document.SourceID != "azure_devops:contoso:iddaa:Bayi-Portal:pr:4225" {+ t.Fatalf("unexpected source id: %s", document.SourceID)+ }+ if !document.UpdatedAt.Equal(closed) {+ t.Fatalf("closed date must win over creation date, got %s", document.UpdatedAt)+ }+ body := string(document.Blob)+ if !strings.Contains(body, "Grace Hopper") || !strings.Contains(body, "Backend tarafina tasindi.") {+ t.Fatalf("pull request body is missing review metadata: %s", body)+ }+ if branch, _ := document.Metadata["source_branch"].(string); branch != "feature/BYS-1517" {+ t.Fatalf("unexpected source branch: %s", branch)+ }+}++func TestIncludeAzureDevOpsItemSkipsUnchangedFingerprint(t *testing.T) {+ item := azureDevOpsItem{Path: "/src/App.cs"}+ item.LatestProcessedChange = &azureDevOpsChange{CommitID: "abc123"}++ request := SyncRequest{Fingerprints: map[string]string{"source-id": "abc123"}}+ if includeAzureDevOpsItem(request, "source-id", item) {+ t.Fatal("an unchanged commit id must not be re-synced")+ }++ request.Fingerprints["source-id"] = "def456"+ if !includeAzureDevOpsItem(request, "source-id", item) {+ t.Fatal("a changed commit id must be re-synced")+ }++ if !includeAzureDevOpsItem(SyncRequest{FromBeginning: true}, "source-id", item) {+ t.Fatal("a full resync must include every item")+ }+}++func TestAzureDevOpsOpenSyncWalksFilesThenPullRequests(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.Contains(r.URL.Path, "/pullrequests"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ case strings.Contains(r.URL.Path, "/items") && r.URL.Query().Get("includeContent") == "true":+ _, _ = io.WriteString(w, "public class App {}")+ case strings.Contains(r.URL.Path, "/items"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"path": "/src/App.cs", "gitObjectType": "blob"},+ map[string]any{"path": "/.gitignore", "gitObjectType": "blob"},+ }})+ default:+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ }})+ }+ }))+ defer server.Close()++ // A batch size of one stops the session at the stage boundary, which is+ // where the resume cursor has to be meaningful.+ connector := newTestAzureDevOpsConnector(t, server.URL, map[string]any{"batch_size": 1})+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: time.Now().UTC()})+ if err != nil {+ t.Fatalf("OpenSync returned error: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("expected the noise file to be skipped, got %d documents", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != "azure_devops:contoso:iddaa:repo-a:file:src/App.cs" {+ t.Fatalf("unexpected source id: %s", batch.Documents[0].SourceID)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor == "" {+ t.Fatal("a batch must carry a resume cursor")+ }++ var cursor azureDevOpsSyncCursor+ if err := json.Unmarshal([]byte(batch.Checkpoint.Cursor), &cursor); err != nil {+ t.Fatalf("cursor is not valid JSON: %v", err)+ }+ if cursor.Stage != azureDevOpsStagePullRequests || cursor.RepoKey != "iddaa/repo-a" {+ t.Fatalf("unexpected cursor: %#v", cursor)+ }++ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("expected EOF after the last repository, got %v", err)+ }+}++func TestAzureDevOpsOpenSyncResumesFromCursor(t *testing.T) {+ listed := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.Contains(r.URL.Path, "/pullrequests"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ case strings.Contains(r.URL.Path, "/items") && r.URL.Query().Get("includeContent") == "true":+ _, _ = io.WriteString(w, "content")+ case strings.Contains(r.URL.Path, "/items"):+ listed+++ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"path": "/a.cs", "gitObjectType": "blob"},+ map[string]any{"path": "/b.cs", "gitObjectType": "blob"},+ }})+ default:+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ }})+ }+ }))+ defer server.Close()++ cursor, _ := json.Marshal(azureDevOpsSyncCursor{RepoKey: "iddaa/repo-a", Stage: azureDevOpsStageCode, FileOffset: 1})+ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ WindowEnd: time.Now().UTC(),+ Resume: &SyncCheckpoint{Cursor: string(cursor)},+ })+ if err != nil {+ t.Fatalf("OpenSync returned error: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(batch.Documents) != 1 || !strings.HasSuffix(batch.Documents[0].SourceID, "file:b.cs") {+ t.Fatalf("resume must continue after the committed file, got %#v", batch.Documents)+ }+ if listed == 0 {+ t.Fatal("the file listing should have been requested once on resume")+ }+}++func TestAzureDevOpsOpenPruneEmitsCodeAndPullRequestIDs(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.Contains(r.URL.Path, "/pullrequests"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"pullRequestId": 7, "title": "fix", "status": "completed"},+ }})+ case strings.Contains(r.URL.Path, "/items"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"path": "/src/App.cs", "gitObjectType": "blob"},+ }})+ default:+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ }})+ }+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune returned error: %v", err)+ }+ defer func() { _ = session.Close() }()++ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(first.Documents) != 1 || !strings.HasSuffix(first.Documents[0].SourceID, "file:src/App.cs") {+ t.Fatalf("unexpected code snapshot: %#v", first.Documents)+ }++ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(second.Documents) != 1 || !strings.HasSuffix(second.Documents[0].SourceID, "pr:7") {+ t.Fatalf("unexpected pull request snapshot: %#v", second.Documents)+ }++ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("expected EOF at the end of the snapshot, got %v", err)+ }+}++func writeAzureDevOpsJSON(t *testing.T, w http.ResponseWriter, payload any) {+ t.Helper()+ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(payload); err != nil {+ t.Fatalf("failed to encode stub payload: %v", err)+ }+}++func TestAzureDevOpsShortPullRequestDescriptionNeedsNoDetailFetch(t *testing.T) {+ if azureDevOpsPullRequestMayBeTruncated(azureDevOpsPullRequest{Description: "kisa aciklama"}) {+ t.Fatal("a short description must not trigger a detail fetch")+ }+ if azureDevOpsPullRequestMayBeTruncated(azureDevOpsPullRequest{}) {+ t.Fatal("an empty description must not trigger a detail fetch")+ }+}++// The pull request list endpoint truncates descriptions at 400 characters.+func TestAzureDevOpsLongPullRequestDescriptionIsRefetchedInFull(t *testing.T) {+ truncated := strings.Repeat("x", azureDevOpsPRDescriptionLimit)+ full := truncated + " ...and the rest of the description"+ detailRequests := 0++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.Contains(r.URL.Path, "/pullrequests/77"):+ detailRequests+++ writeAzureDevOpsJSON(t, w, map[string]any{+ "pullRequestId": 77, "title": "long", "description": full, "status": "completed",+ })+ case strings.Contains(r.URL.Path, "/pullrequests"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"pullRequestId": 77, "title": "long", "description": truncated, "status": "completed"},+ }})+ case strings.Contains(r.URL.Path, "/items"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ default:+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ }})+ }+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, map[string]any{"content_types": azureDevOpsContentPullRequests})+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: time.Now().UTC()})+ if err != nil {+ t.Fatalf("OpenSync returned error: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("expected one pull request document, got %d", len(batch.Documents))+ }+ if !strings.Contains(string(batch.Documents[0].Blob), full) {+ t.Fatal("the document must carry the untruncated description")+ }+ if detailRequests != 1 {+ t.Fatalf("expected exactly one detail fetch, got %d", detailRequests)+ }+}++func TestAzureDevOpsRejectsCleartextCollectionURL(t *testing.T) {+ connector, err := NewAzureDevOpsConnector(map[string]any{+ "organization": "http://tfs.contoso.com/DefaultCollection",+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if err := connector.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "HTTPS") {+ t.Fatalf("cleartext collection URLs must be rejected, got %v", err)+ }+}++func TestAzureDevOpsRejectsUnknownSelectorValues(t *testing.T) {+ for field, value := range map[string]string{"index_mode": "everything", "content_types": "everything"} {+ connector, _ := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ field: value,+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })+ if err := connector.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), "unsupported") {+ t.Fatalf("%s=%q must be rejected, got %v", field, value, err)+ }+ }+}++func TestAzureDevOpsActivePullRequestIsAlwaysReindexed(t *testing.T) {+ windowStart := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)+ created := time.Date(2024, 5, 1, 0, 0, 0, 0, time.UTC)+ request := SyncRequest{WindowStart: &windowStart, WindowEnd: time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)}++ active := azureDevOpsPullRequest{PullRequestID: 1, Status: "active", CreationDate: &created}+ if !includeAzureDevOpsPullRequest(request, "id", active) {+ t.Fatal("an old but still active pull request must be re-indexed; its description can change at any time")+ }++ closed := azureDevOpsPullRequest{PullRequestID: 2, Status: "completed", CreationDate: &created, ClosedDate: &created}+ if includeAzureDevOpsPullRequest(request, "id", closed) {+ t.Fatal("a pull request closed before the window must be skipped")+ }+}++// countingBody reports how many bytes the connector actually pulled off the+// wire, so a regression that reads everything and checks the size afterwards+// cannot pass.+type countingBody struct {+ remaining int64+ read *int64+}++func (b *countingBody) Read(p []byte) (int, error) {+ if b.remaining <= 0 {+ return 0, io.EOF+ }+ n := int64(len(p))+ if n > b.remaining {+ n = b.remaining+ }+ for i := int64(0); i < n; i++ {+ p[i] = 'x'+ }+ b.remaining -= n+ *b.read += n+ return int(n), nil+}++func (b *countingBody) Close() error { return nil }++type stubTransport struct {+ respond func(*http.Request) *http.Response+}++func (t stubTransport) RoundTrip(request *http.Request) (*http.Response, error) {+ return t.respond(request), nil+}++// A single oversized repository file must never be allocated in full.+func TestAzureDevOpsOversizedFileStopsAtTheLimit(t *testing.T) {+ var read int64++ connector, err := NewAzureDevOpsConnector(map[string]any{+ "organization": "contoso",+ "credentials": map[string]any{"azure_devops_pat": "token"},+ })+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ connector.httpClient = &http.Client{Transport: stubTransport{respond: func(*http.Request) *http.Response {+ return &http.Response{+ StatusCode: http.StatusOK,+ Header: http.Header{"Content-Type": []string{"text/plain"}},+ Body: &countingBody{remaining: azureDevOpsMaxFileBytes * 4, read: &read},+ }+ }}}++ repo := azureDevOpsRepository{Project: "iddaa", Name: "repo-a", Branch: "master"}+ content, err := connector.fetchFile(context.Background(), repo, "/huge.bin")+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if content != nil {+ t.Fatalf("an oversized file must be skipped, got %d bytes", len(content))+ }+ if read != azureDevOpsMaxFileBytes+1 {+ t.Fatalf("the download must stop one byte past the limit, read %d bytes", read)+ }+}++func TestAzureDevOpsFileWithinLimitIsReturned(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/plain")+ _, _ = io.WriteString(w, "public class App {}")+ }))+ defer server.Close()++ connector := newTestAzureDevOpsConnector(t, server.URL, nil)+ repo := azureDevOpsRepository{Project: "iddaa", Name: "repo-a", Branch: "master"}++ content, err := connector.fetchFile(context.Background(), repo, "/src/App.cs")+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if string(content) != "public class App {}" {+ t.Fatalf("unexpected content: %q", string(content))+ }+}++func TestAzureDevOpsRetryAfterIsClamped(t *testing.T) {+ response := &http.Response{Header: http.Header{"Retry-After": []string{"86400"}}}+ if got := azureDevOpsRetryAfter(response, time.Second); got != azureDevOpsRetryMaxDelay {+ t.Fatalf("a large Retry-After must be clamped, got %s", got)+ }++ response.Header.Set("Retry-After", "5")+ if got := azureDevOpsRetryAfter(response, time.Second); got != 5*time.Second {+ t.Fatalf("a small Retry-After must be honoured, got %s", got)+ }++ response.Header.Set("Retry-After", "not-a-number")+ if got := azureDevOpsRetryAfter(response, 2*time.Second); got != 2*time.Second {+ t.Fatalf("an unparsable Retry-After must fall back, got %s", got)+ }++ // Multiplying this by time.Second overflows int64; the negative result would+ // slip past the cap and make the retry fire immediately.+ response.Header.Set("Retry-After", strconv.Itoa(math.MaxInt64/int(time.Millisecond)))+ if got := azureDevOpsRetryAfter(response, time.Second); got != azureDevOpsRetryMaxDelay {+ t.Fatalf("an overflowing Retry-After must be clamped, got %s", got)+ }+}++// The remote listing shifts between runs, so an offset alone points at the+// wrong item. The anchor has to decide where the walk continues.+func TestAzureDevOpsResumeFollowsTheAnchorWhenTheListingShifts(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.Contains(r.URL.Path, "/pullrequests"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ case strings.Contains(r.URL.Path, "/items") && r.URL.Query().Get("includeContent") == "true":+ _, _ = io.WriteString(w, "content")+ case strings.Contains(r.URL.Path, "/items"):+ // "added.cs" did not exist when the checkpoint was written, so every+ // index after it has moved by one.+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"path": "/added.cs", "gitObjectType": "blob"},+ map[string]any{"path": "/a.cs", "gitObjectType": "blob"},+ map[string]any{"path": "/b.cs", "gitObjectType": "blob"},+ }})+ default:+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ }})+ }+ }))+ defer server.Close()++ // Written when the listing was [a.cs, b.cs] and a.cs had been committed.+ cursor, _ := json.Marshal(azureDevOpsSyncCursor{+ RepoKey: "iddaa/repo-a",+ Stage: azureDevOpsStageCode,+ FileOffset: 1,+ SourceID: "azure_devops:contoso:iddaa:repo-a:file:a.cs",+ })++ connector := newTestAzureDevOpsConnector(t, server.URL, map[string]any{"batch_size": 1})+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ WindowEnd: time.Now().UTC(),+ Resume: &SyncCheckpoint{Cursor: string(cursor)},+ })+ if err != nil {+ t.Fatalf("OpenSync returned error: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("expected one document, got %d", len(batch.Documents))+ }+ // Offset 1 now points at a.cs, which was already committed. Following the+ // anchor continues at b.cs instead.+ if got := batch.Documents[0].SourceID; got != "azure_devops:contoso:iddaa:repo-a:file:b.cs" {+ t.Fatalf("resume must continue after the anchor, got %s", got)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != batch.Documents[0].SourceID {+ t.Fatalf("the checkpoint must carry the last emitted source id, got %#v", batch.Checkpoint)+ }+}++// Pull request pages shift as new pull requests are opened, so $skip alone is+// not a safe resume position either.+func TestAzureDevOpsPullRequestResumeFollowsTheAnchor(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.Contains(r.URL.Path, "/pullrequests"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"pullRequestId": 3, "title": "newest", "status": "active"},+ map[string]any{"pullRequestId": 2, "title": "committed", "status": "active"},+ map[string]any{"pullRequestId": 1, "title": "pending", "status": "active"},+ }})+ case strings.Contains(r.URL.Path, "/items"):+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{}})+ default:+ writeAzureDevOpsJSON(t, w, map[string]any{"value": []any{+ map[string]any{"name": "repo-a", "defaultBranch": "refs/heads/master", "project": map[string]any{"name": "iddaa"}},+ }})+ }+ }))+ defer server.Close()++ cursor, _ := json.Marshal(azureDevOpsSyncCursor{+ RepoKey: "iddaa/repo-a",+ Stage: azureDevOpsStagePullRequests,+ SourceID: "azure_devops:contoso:iddaa:repo-a:pr:2",+ })++ connector := newTestAzureDevOpsConnector(t, server.URL, map[string]any{"content_types": azureDevOpsContentPullRequests})+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ WindowEnd: time.Now().UTC(),+ Resume: &SyncCheckpoint{Cursor: string(cursor)},+ })+ if err != nil {+ t.Fatalf("OpenSync returned error: %v", err)+ }+ defer func() { _ = session.Close() }()++ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch returned error: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("only the pull request after the anchor should be emitted, got %d", len(batch.Documents))+ }+ if got := batch.Documents[0].SourceID; got != "azure_devops:contoso:iddaa:repo-a:pr:1" {+ t.Fatalf("unexpected document: %s", got)+ }+}diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 5f44befdbb..eb4fb352f2 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -30,6 +30,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "rss", NewRSSConnector)registerBuiltIn(registry, "salesforce", NewSalesforceConnector)registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)+ registerBuiltIn(registry, "azure_devops", NewAzureDevOpsConnector)registerBuiltIn(registry, "dropbox", NewDropboxConnector)registerBuiltIn(registry, "github", NewGitHubConnector)registerBuiltIn(registry, "gitlab", NewGitlabConnector)diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex a7a847e934..d12aa6873d 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -78,6 +78,7 @@ from common.data_source.box_connector import BoxConnectorfrom common.data_source.github.connector import GithubConnectorfrom common.data_source.gitlab_connector import GitlabConnectorfrom common.data_source.bitbucket.connector import BitbucketConnector+from common.data_source.azure_devops.connector import AzureDevOpsConnectorfrom common.data_source.interfaces import CheckpointOutputWrapperfrom common.data_source.exceptions import ConnectorValidationErrorfrom common.log_utils import init_root_logger@@ -1875,6 +1876,70 @@ class Bitbucket(SyncBase):return wrapper()+class AzureDevOps(SyncBase):+ SOURCE_NAME: str = FileSource.AZURE_DEVOPS++ async def _generate(self, task: dict):+ self.connector = AzureDevOpsConnector(+ organization=self.conf.get("organization"),+ index_mode=self.conf.get("index_mode") or "organization",+ projects=self.conf.get("projects"),+ repositories=self.conf.get("repositories"),+ content_types=self.conf.get("content_types") or "both",+ )++ self.connector.load_credentials(+ {+ "azure_devops_pat": self.conf["credentials"].get("azure_devops_pat"),+ }+ )++ if task["reindex"] == "1" or not task["poll_range_start"]:+ start_time = datetime.fromtimestamp(0, tz=timezone.utc)+ _begin_info = "totally"+ else:+ start_time = task.get("poll_range_start")+ _begin_info = f"from {start_time}"++ end_time = datetime.now(timezone.utc)++ def document_batches():+ checkpoint = self.connector.build_dummy_checkpoint()+ iterations = 0+ iteration_limit = 100_000++ while checkpoint.has_more:+ iterations += 1+ if iterations > iteration_limit:+ logging.error("Azure DevOps sync exceeded %d iterations; aborting to avoid an endless loop.", iteration_limit)+ # Breaking here would end the generator normally and the task+ # would be recorded as a successful, complete sync.+ raise RuntimeError(f"Azure DevOps sync exceeded {iteration_limit} iterations before completing.")++ gen = self.connector.load_from_checkpoint(start=start_time.timestamp(), end=end_time.timestamp(), checkpoint=checkpoint)++ while True:+ try:+ item = next(gen)+ if isinstance(item, ConnectorFailure):+ # A failed document must not cost the checkpoint: keep consuming+ # so the generator returns its updated resume position, otherwise+ # a deterministic failure replays the same offset forever.+ logging.error("Azure DevOps connector failure: %s", item.failure_message)+ continue+ yield [item]+ except StopIteration as e:+ checkpoint = e.value+ break++ def wrapper():+ for batch in document_batches():+ yield batch++ self.log_connection("AzureDevOps", f"organization({self.conf.get('organization')})", task)+ return wrapper()++class SeaFile(SyncBase):SOURCE_NAME: str = FileSource.SEAFILE@@ -2162,6 +2227,7 @@ func_factory = {FileSource.GITHUB: Github,FileSource.GITLAB: Gitlab,FileSource.BITBUCKET: Bitbucket,+ FileSource.AZURE_DEVOPS: AzureDevOps,FileSource.SEAFILE: SeaFile,FileSource.MYSQL: MySQL,FileSource.POSTGRESQL: PostgreSQL,diff --git a/test/unit_test/data_source/test_azure_devops_connector_unit.py b/test/unit_test/data_source/test_azure_devops_connector_unit.pynew file mode 100644index 0000000000..f388dd54fa--- /dev/null+++ b/test/unit_test/data_source/test_azure_devops_connector_unit.py@@ -0,0 +1,750 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib.util+import sys+from dataclasses import dataclass, field+from datetime import datetime, timezone+from pathlib import Path+from types import ModuleType+from typing import Any, Generic, TypeVar++import pytest+from pydantic import BaseModel++T = TypeVar("T")+++@dataclass+class _BasicExpertInfo:+ display_name: str | None = None+++@dataclass+class _Document:+ id: str+ blob: bytes+ source: str+ semantic_identifier: str+ extension: str+ doc_updated_at: datetime+ size_bytes: int+ primary_owners: list = field(default_factory=list)+ metadata: dict | None = None+++@dataclass+class _SlimDocument:+ id: str+++@dataclass+class _DocumentFailure:+ document_id: str+ document_link: str | None = None+++@dataclass+class _ConnectorFailure:+ failed_document: Any = None+ failure_message: str = ""+ exception: Exception | None = None+++class _ConnectorCheckpoint(BaseModel):+ has_more: bool = True+++class _CheckpointedConnector(Generic[T]):+ pass+++class _SlimConnectorWithPermSync:+ pass+++class _DocumentSource:+ AZURE_DEVOPS = type("_Value", (), {"value": "azure_devops"})()+++def _install_dependency_stubs():+ config_module = ModuleType("common.data_source.config")+ config_module.DocumentSource = _DocumentSource+ config_module.INDEX_BATCH_SIZE = 10+ config_module.REQUEST_TIMEOUT_SECONDS = 30++ exceptions_module = ModuleType("common.data_source.exceptions")+ for name in (+ "ConnectorMissingCredentialError",+ "ConnectorValidationError",+ "CredentialExpiredError",+ "InsufficientPermissionsError",+ "UnexpectedValidationError",+ ):+ setattr(exceptions_module, name, type(name, (Exception,), {}))++ interfaces_module = ModuleType("common.data_source.interfaces")+ interfaces_module.CheckpointedConnector = _CheckpointedConnector+ interfaces_module.CheckpointOutput = object+ interfaces_module.IndexingHeartbeatInterface = object+ interfaces_module.SecondsSinceUnixEpoch = float+ interfaces_module.SlimConnectorWithPermSync = _SlimConnectorWithPermSync++ models_module = ModuleType("common.data_source.models")+ models_module.BasicExpertInfo = _BasicExpertInfo+ models_module.ConnectorCheckpoint = _ConnectorCheckpoint+ models_module.ConnectorFailure = _ConnectorFailure+ models_module.Document = _Document+ models_module.DocumentFailure = _DocumentFailure+ models_module.SlimDocument = _SlimDocument++ utils_module = ModuleType("common.data_source.utils")+ utils_module.get_file_ext = lambda file_name: Path(file_name).suffix.lower()++ cross_module = ModuleType("common.data_source.cross_connector_utils")+ cross_module.__path__ = []+ retry_module = ModuleType("common.data_source.cross_connector_utils.retry_wrapper")+ retry_module.retry_builder = lambda **kwargs: lambda func: func+ rate_limit_module = ModuleType("common.data_source.cross_connector_utils.rate_limit_wrapper")+ rate_limit_module.rate_limit_builder = lambda **kwargs: lambda func: func++ sys.modules["common.data_source.config"] = config_module+ sys.modules["common.data_source.exceptions"] = exceptions_module+ sys.modules["common.data_source.interfaces"] = interfaces_module+ sys.modules["common.data_source.models"] = models_module+ sys.modules["common.data_source.utils"] = utils_module+ sys.modules["common.data_source.cross_connector_utils"] = cross_module+ sys.modules["common.data_source.cross_connector_utils.retry_wrapper"] = retry_module+ sys.modules["common.data_source.cross_connector_utils.rate_limit_wrapper"] = rate_limit_module+++def _load_azure_devops_modules():+ """Load the Azure DevOps connector in isolation (avoid the package __init__)."""+ repo_root = Path(__file__).resolve().parents[3]+ package_name = "common.data_source"+ saved_modules = {name: module for name, module in sys.modules.items() if name == package_name or name.startswith(f"{package_name}.")}++ package_stub = ModuleType(package_name)+ package_stub.__path__ = [str(repo_root / "common" / "data_source")]+ sys.modules[package_name] = package_stub++ subpackage = ModuleType(f"{package_name}.azure_devops")+ subpackage.__path__ = [str(repo_root / "common" / "data_source" / "azure_devops")]+ sys.modules[f"{package_name}.azure_devops"] = subpackage++ _install_dependency_stubs()++ def _load(module_name: str, relative_path: str):+ spec = importlib.util.spec_from_file_location(module_name, repo_root / relative_path)+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ sys.modules[module_name] = module+ spec.loader.exec_module(module)+ return module++ try:+ utils = _load(f"{package_name}.azure_devops.utils", "common/data_source/azure_devops/utils.py")+ connector = _load(f"{package_name}.azure_devops.connector", "common/data_source/azure_devops/connector.py")+ # The connector raises these exact stub classes, so they are captured+ # before sys.modules is restored.+ exceptions = sys.modules[f"{package_name}.exceptions"]+ return utils, connector, exceptions+ finally:+ for name in list(sys.modules):+ if name == package_name or name.startswith(f"{package_name}."):+ if name in saved_modules:+ sys.modules[name] = saved_modules[name]+ else:+ sys.modules.pop(name, None)+++azure_utils, azure_connector, azure_exceptions = _load_azure_devops_modules()++ConnectorMissingCredentialError = azure_exceptions.ConnectorMissingCredentialError+CredentialExpiredError = azure_exceptions.CredentialExpiredError+InsufficientPermissionsError = azure_exceptions.InsufficientPermissionsError+UnexpectedValidationError = azure_exceptions.UnexpectedValidationError++AzureDevOpsConnector = azure_connector.AzureDevOpsConnector+INDEX_MODE_ORGANIZATION = azure_connector.INDEX_MODE_ORGANIZATION+INDEX_MODE_PROJECTS = azure_connector.INDEX_MODE_PROJECTS+INDEX_MODE_REPOSITORIES = azure_connector.INDEX_MODE_REPOSITORIES+++class _FakeResponse:+ def __init__(self, payload: Any = None, status_code: int = 200, content_type: str = "application/json", text: str = ""):+ self._payload = payload if payload is not None else {}+ self.status_code = status_code+ self.headers = {"content-type": content_type}+ self.content = text.encode("utf-8") if text else b""++ def json(self) -> Any:+ return self._payload+++class _FakeClient:+ """Matches request URLs by substring and returns canned payloads."""++ def __init__(self, routes: dict[str, Any]):+ self.routes = routes+ self.requests: list[str] = []++ def get(self, url: str, params: dict[str, Any] | None = None, timeout: Any = None) -> _FakeResponse:+ self.requests.append(url)+ for fragment, response in self.routes.items():+ if fragment in url:+ return response if isinstance(response, _FakeResponse) else _FakeResponse(response)+ return _FakeResponse({}, status_code=404)++ def __enter__(self) -> "_FakeClient":+ return self++ def __exit__(self, *args) -> None:+ return None+++class _FakeStreamResponse:+ """Minimal stand-in for a streamed httpx response."""++ def __init__(self, chunks: list[bytes], status_code: int = 200, content_type: str = "text/plain"):+ self._chunks = chunks+ self.status_code = status_code+ self.headers = {"content-type": content_type}+ self.consumed_chunks = 0+ self.consumed_bytes = 0++ def iter_bytes(self):+ for chunk in self._chunks:+ self.consumed_chunks += 1+ self.consumed_bytes += len(chunk)+ yield chunk++ def __enter__(self) -> "_FakeStreamResponse":+ return self++ def __exit__(self, *args) -> None:+ return None+++class _FakeStreamClient:+ def __init__(self, response: _FakeStreamResponse):+ self._response = response+ self.consumed = 0++ def stream(self, method: str, url: str, params=None, timeout=None) -> _FakeStreamResponse:+ return self._response+++def _build_connector(**kwargs) -> AzureDevOpsConnector:+ connector = AzureDevOpsConnector(organization=kwargs.pop("organization", "contoso"), **kwargs)+ connector.load_credentials({"azure_devops_pat": "token"})+ return connector+++def test_organization_url_accepts_bare_name():+ assert azure_utils.organization_url("contoso") == "https://dev.azure.com/contoso"+++def test_organization_url_accepts_self_hosted_collection():+ """Azure DevOps Server lives behind a custom collection URL, not dev.azure.com."""+ assert azure_utils.organization_url("https://tfs.contoso.com/DefaultCollection/") == "https://tfs.contoso.com/DefaultCollection"+++def test_invalid_token_returns_203_sign_in_page_not_401():+ """Azure DevOps answers a bad PAT with 203 and an HTML sign-in page.++ Without this mapping the failure surfaces as an unrelated JSON parse error.+ """+ with pytest.raises(CredentialExpiredError) as excinfo:+ azure_utils.raise_for_auth(_FakeResponse(status_code=203, content_type="text/html"))+ assert "personal access token" in str(excinfo.value).lower()+++def test_html_response_with_200_is_still_treated_as_sign_in_page():+ with pytest.raises(CredentialExpiredError):+ azure_utils.raise_for_auth(_FakeResponse(status_code=200, content_type="text/html; charset=utf-8"))+++def test_forbidden_response_reports_missing_code_read_scope():+ with pytest.raises(InsufficientPermissionsError) as excinfo:+ azure_utils.raise_for_auth(_FakeResponse(status_code=403))+ assert "Code (Read)" in str(excinfo.value)+++def test_successful_response_passes_through():+ assert azure_utils.raise_for_auth(_FakeResponse(status_code=200)) is None+++def test_throttling_response_is_retriable():+ """Azure DevOps throttles on throughput units; 429 must not end the sync."""+ client = _FakeClient({"": _FakeResponse(status_code=429)})+ with pytest.raises(azure_utils.AzureDevOpsRetriableError):+ azure_utils.azure_devops_get(client, "https://dev.azure.com/contoso/_apis/projects")+++def test_retry_after_is_clamped(monkeypatch):+ """A large Retry-After must not park a sync worker for hours."""+ slept: list[float] = []+ monkeypatch.setattr(azure_utils.time, "sleep", lambda seconds: slept.append(seconds))++ azure_utils.sleep_for_retry_after("86400")+ azure_utils.sleep_for_retry_after("5")+ azure_utils.sleep_for_retry_after("not-a-number")+ azure_utils.sleep_for_retry_after(None)++ assert slept == [azure_utils.MAX_RETRY_AFTER_SECONDS, 5]+++def test_server_error_is_retriable():+ client = _FakeClient({"": _FakeResponse(status_code=503)})+ with pytest.raises(azure_utils.AzureDevOpsRetriableError):+ azure_utils.azure_devops_get(client, "https://dev.azure.com/contoso/_apis/projects")+++def test_client_error_is_not_retriable():+ client = _FakeClient({"": _FakeResponse(status_code=404)})+ with pytest.raises(azure_utils.AzureDevOpsNonRetriableError):+ azure_utils.azure_devops_get(client, "https://dev.azure.com/contoso/_apis/projects")+++ "path,expected",+ [+ ("/src/Payments/RefundService.cs", False),+ ("/Dockerfile", False),+ ("/LICENSE", False),+ ("/.gitattributes", True),+ ("/.gitignore", True),+ ("/node_modules/lib/index.js", True),+ ("/Bayi-Portal.Api/bin/Debug/app.dll", True),+ ("/docs/logo.png", True),+ ],+)+def test_should_skip_path(path: str, expected: bool):+ assert azure_utils.should_skip_path(path) is expected+++def test_document_extension_falls_back_to_txt_for_extensionless_files():+ """RAGFlow dispatches parsing on the extension, so it must never be empty."""+ assert azure_utils.document_extension("Dockerfile") == ".txt"+ assert azure_utils.document_extension("src/App.cs") == ".cs"+++def test_oversized_file_is_abandoned_mid_download():+ """The limit is enforced while reading, not after the body is in memory."""+ chunk = b"x" * 64_000+ chunks = [chunk] * (azure_utils.MAX_FILE_BYTES // len(chunk) + 4)+ response = _FakeStreamResponse(chunks)+ client = _FakeStreamClient(response)++ content = azure_utils.fetch_file_content(client, "https://dev.azure.com/contoso/p/_apis/git/repositories/r", "/huge.bin", "master")++ assert content is None+ # Reading the whole body and checking the size afterwards would also return+ # None, so the download itself has to be shown to stop early.+ assert response.consumed_chunks < len(chunks)+ assert response.consumed_bytes <= azure_utils.MAX_FILE_BYTES + len(chunk)+++def test_file_within_limit_is_returned():+ client = _FakeStreamClient(_FakeStreamResponse([b"public class ", b"App {}"]))+ content = azure_utils.fetch_file_content(client, "https://dev.azure.com/contoso/p/_apis/git/repositories/r", "/src/App.cs", "master")+ assert content == b"public class App {}"+++def test_default_branch_strips_ref_prefix():+ assert azure_utils.default_branch_of({"defaultBranch": "refs/heads/master"}) == "master"+ assert azure_utils.default_branch_of({}) == "main"+++def test_cleartext_collection_url_is_rejected():+ """The PAT travels in the Authorization header, so HTTP is refused."""+ with pytest.raises(UnexpectedValidationError) as excinfo:+ azure_utils.organization_url("http://tfs.contoso.com/DefaultCollection")+ assert "HTTPS" in str(excinfo.value)+++def test_html_body_is_not_an_auth_failure_for_raw_content():+ """A repository may legitimately contain .html files."""+ response = _FakeResponse(status_code=200, content_type="text/html", text="<html>page</html>")+ assert azure_utils.raise_for_auth(response, expect_json=False) is None+++ "overrides",+ [+ {"index_mode": "everything"},+ {"content_types": "everything"},+ {"index_mode": INDEX_MODE_REPOSITORIES},+ ],+)+def test_unusable_settings_are_rejected_before_any_request(overrides):+ connector = _build_connector(**overrides)+ with pytest.raises(UnexpectedValidationError, match="(?i)unsupported|required"):+ connector._validate_settings()+++def test_active_pull_request_is_always_reindexed():+ """Azure DevOps has no dependable "updated" timestamp for pull requests."""+ window_start = datetime(2026, 1, 1, tzinfo=timezone.utc)+ window_end = datetime(2026, 2, 1, tzinfo=timezone.utc)++ active = {"status": "active", "creationDate": "2024-05-01T00:00:00Z"}+ assert azure_utils.pull_request_in_window(active, window_start, window_end) is True++ closed_before = {"status": "completed", "closedDate": "2024-05-01T00:00:00Z"}+ assert azure_utils.pull_request_in_window(closed_before, window_start, window_end) is False++ closed_inside = {"status": "completed", "closedDate": "2026-01-15T00:00:00Z"}+ assert azure_utils.pull_request_in_window(closed_inside, window_start, window_end) is True+++def test_load_credentials_requires_personal_access_token():+ connector = AzureDevOpsConnector(organization="contoso")+ with pytest.raises(ConnectorMissingCredentialError):+ connector.load_credentials({})+++def test_repository_filter_accepts_qualified_and_bare_names():+ connector = _build_connector(index_mode=INDEX_MODE_REPOSITORIES, repositories="iddaa/Bayi-Portal, sportsbook")+ assert connector._matches_repository_filter("iddaa", "Bayi-Portal") is True+ assert connector._matches_repository_filter("other", "sportsbook") is True+ assert connector._matches_repository_filter("iddaa", "unrelated") is False+++def test_repository_filter_is_inactive_for_organization_scope():+ connector = _build_connector(index_mode=INDEX_MODE_ORGANIZATION, repositories="ignored")+ assert connector._matches_repository_filter("any", "repository") is True+++def test_discover_repositories_skips_disabled_and_sorts():+ client = _FakeClient(+ {+ "_apis/git/repositories": {+ "value": [+ {"name": "zeta", "project": {"name": "iddaa"}, "defaultBranch": "refs/heads/master"},+ {"name": "alpha", "project": {"name": "iddaa"}, "defaultBranch": "refs/heads/develop"},+ {"name": "retired", "project": {"name": "iddaa"}, "isDisabled": True},+ ]+ }+ }+ )+ connector = _build_connector()+ repos = connector._discover_repositories(client)++ assert [repo["name"] for repo in repos] == ["alpha", "zeta"]+ assert repos[0]["branch"] == "develop"+++def test_list_items_filters_folders_and_noise():+ client = _FakeClient(+ {+ "/items": {+ "value": [+ {"path": "/src", "gitObjectType": "tree", "isFolder": True},+ {"path": "/src/App.cs", "gitObjectType": "blob"},+ {"path": "/.gitignore", "gitObjectType": "blob"},+ {"path": "/node_modules/a.js", "gitObjectType": "blob"},+ ]+ }+ }+ )+ items = azure_utils.list_items(client, "https://dev.azure.com/contoso/iddaa/_apis/git/repositories/repo", "master")+ assert [item["path"] for item in items] == ["/src/App.cs"]+++def test_map_item_to_document_carries_path_commit_and_web_url():+ item = {+ "path": "/src/Payments/RefundService.cs",+ "latestProcessedChange": {+ "commitId": "abc123",+ "committer": {"name": "Ada Lovelace", "date": "2026-01-13T21:53:06Z"},+ },+ }+ document = azure_utils.map_item_to_document(+ item,+ b"public class RefundService {}",+ "contoso",+ "https://dev.azure.com/contoso",+ "iddaa",+ "Bayi-Portal",+ "master",+ )++ assert document.id == "azure_devops:contoso:iddaa:Bayi-Portal:file:src/Payments/RefundService.cs"+ assert document.extension == ".cs"+ assert document.size_bytes == len(b"public class RefundService {}")+ assert document.doc_updated_at == datetime(2026, 1, 13, 21, 53, 6, tzinfo=timezone.utc)+ assert document.metadata["commit_id"] == "abc123"+ assert document.metadata["repository"] == "Bayi-Portal"+ assert "path=/src/Payments/RefundService.cs" in document.metadata["web_url"]+ assert document.primary_owners[0].display_name == "Ada Lovelace"+++def test_map_item_to_document_survives_missing_commit_metadata():+ document = azure_utils.map_item_to_document({"path": "/README.md"}, b"# readme", "contoso", "https://dev.azure.com/contoso", "iddaa", "repo", "master")+ assert document.doc_updated_at is not None+ assert document.primary_owners == []+++def test_map_pull_request_to_document_summarises_review_metadata():+ pull_request = {+ "pullRequestId": 4225,+ "title": "BYS-1517 - PDF olusturma islemi",+ "description": "Backend tarafina tasindi.",+ "status": "completed",+ "createdBy": {"displayName": "Ada Lovelace"},+ "reviewers": [{"displayName": "Grace Hopper"}],+ "sourceRefName": "refs/heads/feature/BYS-1517",+ "targetRefName": "refs/heads/master",+ "creationDate": "2025-11-18T06:20:07.836003Z",+ "closedDate": "2025-11-20T06:20:07.836003Z",+ }+ document = azure_utils.map_pull_request_to_document(pull_request, "contoso", "https://dev.azure.com/contoso", "iddaa", "Bayi-Portal")++ assert document.id == "azure_devops:contoso:iddaa:Bayi-Portal:pr:4225"+ assert document.semantic_identifier.startswith("PR #4225:")+ assert document.extension == ".txt"+ assert document.metadata["status"] == "completed"+ assert document.metadata["source_branch"] == "feature/BYS-1517"+ assert document.doc_updated_at == datetime(2025, 11, 20, 6, 20, 7, 836003, tzinfo=timezone.utc)++ body = document.blob.decode("utf-8")+ assert "Grace Hopper" in body+ assert "Backend tarafina tasindi." in body+++def test_short_pull_request_description_needs_no_detail_fetch():+ assert azure_utils.pull_request_may_be_truncated({"description": "kisa aciklama"}) is False+ assert azure_utils.pull_request_may_be_truncated({}) is False+++def test_long_pull_request_description_is_refetched_in_full(monkeypatch):+ """The list endpoint truncates descriptions at 400 characters."""+ truncated = "x" * azure_utils.PULL_REQUEST_DESCRIPTION_LIMIT+ full = truncated + " ...and the rest of the description"++ assert azure_utils.pull_request_may_be_truncated({"description": truncated}) is True++ client = _FakeClient(+ {+ "/items": {"value": []},+ "/pullrequests/77": {"pullRequestId": 77, "title": "long", "description": full, "status": "completed"},+ "/pullrequests": {"value": [{"pullRequestId": 77, "title": "long", "description": truncated, "status": "completed"}]},+ "_apis/git/repositories": {"value": [{"name": "repo-a", "project": {"name": "iddaa"}, "defaultBranch": "refs/heads/master"}]},+ }+ )+ connector = _build_connector(content_types="pull_requests")+ monkeypatch.setattr(connector, "_client", lambda: client)++ checkpoint = connector.build_dummy_checkpoint()+ documents = []+ while checkpoint.has_more:+ generator = connector.load_from_checkpoint(start=0.0, end=datetime.now(timezone.utc).timestamp(), checkpoint=checkpoint)+ while True:+ try:+ documents.append(next(generator))+ except StopIteration as stop:+ checkpoint = stop.value+ break++ assert len(documents) == 1+ assert full in documents[0].blob.decode("utf-8")+ assert any("/pullrequests/77" in request for request in client.requests)+++def test_pull_request_updated_at_prefers_closed_date():+ assert azure_utils.pull_request_updated_at({"creationDate": "2025-01-01T00:00:00Z", "closedDate": "2025-02-01T00:00:00Z"}) == datetime(2025, 2, 1, tzinfo=timezone.utc)+ assert azure_utils.pull_request_updated_at({}) is None+++def test_checkpoint_walks_files_then_pull_requests_then_next_repo(monkeypatch):+ """One stage per invocation; the checkpoint records the exact resume position."""+ # Routes are matched in order, so the specific paths must precede the+ # repository collection URL they are nested under.+ client = _FakeClient(+ {+ "/items": {"value": [{"path": "/src/App.cs", "gitObjectType": "blob"}]},+ "/pullrequests": {"value": []},+ "_apis/git/repositories": {"value": [{"name": "repo-a", "project": {"name": "iddaa"}, "defaultBranch": "refs/heads/master"}]},+ }+ )+ connector = _build_connector()+ monkeypatch.setattr(connector, "_client", lambda: client)+ monkeypatch.setattr(azure_utils, "fetch_file_content", lambda *args, **kwargs: b"code")+ monkeypatch.setattr(azure_connector, "fetch_file_content", lambda *args, **kwargs: b"code")++ checkpoint = connector.build_dummy_checkpoint()+ start, end = 0.0, datetime.now(timezone.utc).timestamp()++ generator = connector.load_from_checkpoint(start=start, end=end, checkpoint=checkpoint)+ documents = []+ while True:+ try:+ documents.append(next(generator))+ except StopIteration as stop:+ checkpoint = stop.value+ break++ assert [document.id for document in documents] == ["azure_devops:contoso:iddaa:repo-a:file:src/App.cs"]+ assert checkpoint.stage == "pull_requests"+ assert checkpoint.has_more is True++ generator = connector.load_from_checkpoint(start=start, end=end, checkpoint=checkpoint)+ while True:+ try:+ next(generator)+ except StopIteration as stop:+ checkpoint = stop.value+ break++ assert checkpoint.current_repo_index == 1+ assert checkpoint.has_more is False+++def _drain(connector, checkpoint):+ """Run the connector to completion, collecting documents and failures."""+ produced = []+ while checkpoint.has_more:+ generator = connector.load_from_checkpoint(start=0.0, end=datetime.now(timezone.utc).timestamp(), checkpoint=checkpoint)+ while True:+ try:+ produced.append(next(generator))+ except StopIteration as stop:+ checkpoint = stop.value+ break+ return produced+++def test_resume_follows_the_anchor_when_the_listing_shifts(monkeypatch):+ """An offset points into a listing that shifts; the anchor does not."""+ client = _FakeClient(+ {+ # "added.cs" did not exist when the checkpoint was written, so every+ # index after it has moved by one.+ "/items": {+ "value": [+ {"path": "/added.cs", "gitObjectType": "blob"},+ {"path": "/a.cs", "gitObjectType": "blob"},+ {"path": "/b.cs", "gitObjectType": "blob"},+ ]+ },+ "/pullrequests": {"value": []},+ "_apis/git/repositories": {"value": [{"name": "repo-a", "project": {"name": "iddaa"}, "defaultBranch": "refs/heads/master"}]},+ }+ )+ connector = _build_connector(content_types="code")+ monkeypatch.setattr(connector, "_client", lambda: client)+ monkeypatch.setattr(azure_connector, "fetch_file_content", lambda *args, **kwargs: b"code")++ checkpoint = connector.build_dummy_checkpoint()+ checkpoint.repos_queue = [{"project": "iddaa", "name": "repo-a", "branch": "master"}]+ checkpoint.file_offset = 1+ checkpoint.last_source_id = "azure_devops:contoso:iddaa:repo-a:file:a.cs"++ produced = _drain(connector, checkpoint)++ # Offset 1 now points at a.cs, which was already committed. Following the+ # anchor continues at b.cs instead.+ assert [document.id for document in produced] == ["azure_devops:contoso:iddaa:repo-a:file:b.cs"]+++def test_failed_file_is_retried_once_before_the_stage_ends(monkeypatch):+ """Advancing past a failure without a retry would drop the file silently."""+ attempts: list[str] = []++ def flaky_fetch(client, repo_api_url, path, branch):+ attempts.append(path)+ if path == "/b.cs" and attempts.count("/b.cs") == 1:+ raise RuntimeError("transient failure")+ return b"code"++ client = _FakeClient(+ {+ "/items": {+ "value": [+ {"path": "/a.cs", "gitObjectType": "blob"},+ {"path": "/b.cs", "gitObjectType": "blob"},+ ]+ },+ "/pullrequests": {"value": []},+ "_apis/git/repositories": {"value": [{"name": "repo-a", "project": {"name": "iddaa"}, "defaultBranch": "refs/heads/master"}]},+ }+ )+ connector = _build_connector(content_types="code")+ monkeypatch.setattr(connector, "_client", lambda: client)+ monkeypatch.setattr(azure_connector, "fetch_file_content", flaky_fetch)++ produced = _drain(connector, connector.build_dummy_checkpoint())++ failures = [item for item in produced if hasattr(item, "failure_message")]+ documents = [item for item in produced if hasattr(item, "id")]++ assert len(failures) == 1+ assert attempts.count("/b.cs") == 2, "the failed file must be attempted exactly twice"+ assert "azure_devops:contoso:iddaa:repo-a:file:b.cs" in [document.id for document in documents]+++def test_checkpoint_round_trips_through_json():+ connector = _build_connector()+ checkpoint = connector.build_dummy_checkpoint()+ checkpoint.repos_queue = [{"project": "iddaa", "name": "repo-a", "branch": "master"}]+ checkpoint.file_offset = 500++ restored = connector.validate_checkpoint_json(checkpoint.model_dump_json())+ assert restored.file_offset == 500+ assert restored.repos_queue[0]["name"] == "repo-a"+++def test_content_types_control_which_families_are_indexed():+ assert _build_connector(content_types="code")._indexes_pull_requests() is False+ assert _build_connector(content_types="pull_requests")._indexes_code() is False+ both = _build_connector(content_types="both")+ assert both._indexes_code() and both._indexes_pull_requests()diff --git a/web/src/assets/svg/data-source/azure-devops.svg b/web/src/assets/svg/data-source/azure-devops.svgnew file mode 100644index 0000000000..7a3e2282cd--- /dev/null+++ b/web/src/assets/svg/data-source/azure-devops.svg@@ -0,0 +1,4 @@+<svg xmlns="http://www.w3.org/2000/svg" aria-label="Azure DevOps" role="img" viewBox="0 0 512 512">+ <rect width="512" height="512" rx="15%" fill="#ffffff"/>+ <path d="M190 196c-46 0-76 27-76 60s30 60 76 60c40 0 62-33 66-42 4 9 26 42 66 42 46 0 76-27 76-60s-30-60-76-60c-40 0-62 33-66 42-4-9-26-42-66-42zm0 34c22 0 40 12 40 26s-18 26-40 26-40-12-40-26 18-26 40-26zm132 0c22 0 40 12 40 26s-18 26-40 26-40-12-40-26 18-26 40-26z" fill="#0078d4"/>+</svg>diff --git a/web/src/locales/ar.ts b/web/src/locales/ar.tsindex 7554fa89fa..f581dba01b 100644--- a/web/src/locales/ar.ts+++ b/web/src/locales/ar.ts@@ -923,6 +923,7 @@ export default {'Upload the OAuth JSON generated from Google Console. If it only contains client credentials, run the browser-based verification once to mint long-lived refresh tokens.',dropboxDescription:'Connect your Dropbox to sync files and folders from a chosen account.',+ azure_devopsDescription: 'اربط Azure DevOps لمزامنة ملفات المستودع وطلبات السحب.',bitbucketDescription: 'Connect Bitbucket to sync PR content.',bitbucketTopWorkspaceTip:'The Bitbucket workspace to index (e.g., "atlassian" from https://bitbucket.org/atlassian/workspace ).',diff --git a/web/src/locales/bg.ts b/web/src/locales/bg.tsindex 978936f5ab..323db38920 100644--- a/web/src/locales/bg.ts+++ b/web/src/locales/bg.ts@@ -1000,6 +1000,7 @@ The above is the content you need to summarize.`,gmailTokenTip: 'Качете OAuth JSON, генериран от Google Console.',dropboxDescription:'Свържете вашия Dropbox за синхронизиране на файлове и папки от избран акаунт.',+ azure_devopsDescription: 'Свържете Azure DevOps, за да синхронизирате файловете на хранилището и заявките за изтегляне (pull requests).',bitbucketDescription:'Свържете Bitbucket за синхронизиране на PR съдържание.',bitbucketTopWorkspaceTip:diff --git a/web/src/locales/de.ts b/web/src/locales/de.tsindex a855bc4481..70b5ec4b6e 100644--- a/web/src/locales/de.ts+++ b/web/src/locales/de.ts@@ -1014,6 +1014,7 @@ Beispiel: Virtual Hosted Style`,'Laden Sie das OAuth-JSON hoch, das von der Google Console generiert wurde. Wenn es nur Client-Anmeldeinformationen enthält, führen Sie die browserbasierte Überprüfung einmal durch, um langlebige Refresh-Token zu erstellen.',dropboxDescription:'Verbinden Sie Ihre Dropbox, um Dateien und Ordner von einem ausgewählten Konto zu synchronisieren.',+ azure_devopsDescription: 'Verbinden Sie Azure DevOps, um Repository-Dateien und Pull Requests zu synchronisieren.',bitbucketDescription:'Bitbucket verbinden, um PR-Inhalte zu synchronisieren.',zendeskDescription:diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 8e8656835a..2c80a790f6 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1397,6 +1397,14 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-sdataSourceFieldWikiBaseUrl: 'Wiki Base URL',dataSourceFieldIsCloud: 'Is Cloud',dataSourceFieldIndexMode: 'Index Mode',+ dataSourceFieldAzureDevOpsPat: 'Azure DevOps personal access token',+ dataSourceFieldAzureDevOpsOrganization: 'Azure DevOps organization',+ dataSourceFieldAzureDevOpsRepositories: 'Repositories',+ dataSourceFieldAzureDevOpsContentTypes: 'Content types',+ dataSourceOptionOrganization: 'Organization',+ dataSourceOptionCode: 'Code',+ dataSourceOptionPullRequests: 'Pull requests',+ dataSourceOptionBoth: 'Both',dataSourceFieldPageId: 'Page ID',dataSourceFieldSpaceKey: 'Space Key',dataSourceFieldIndexRecursively: 'Index Recursively',@@ -1576,6 +1584,13 @@ Example: Virtual Hosted Style`,'Connect a SharePoint site via Microsoft Graph to sync its document libraries.',sharepointSiteUrlTip:'Full URL of the SharePoint site to index, e.g. https://contoso.sharepoint.com/sites/MySite. Requires an Azure AD app with Sites.Read.All and Files.Read.All application permissions (admin consent).',+ azureDevOpsPatTip: 'A personal access token with the Code (Read) scope.',+ azureDevOpsOrganizationTip: 'Organization name (e.g. "contoso"), or the full collection URL of a self-hosted Azure DevOps Server (e.g. https://tfs.contoso.com/DefaultCollection).',+ azureDevOpsProjectsTip: 'Comma separated team project names. E.g., Project1,Project2',+ azureDevOpsRepositoriesTip: 'Comma separated repositories. Use project/repo to disambiguate repositories that share a name.',+ azureDevOpsOrganizationScopeTip: 'Every repository visible to the token in this organization will be indexed.',+ azureDevOpsContentTypesTip: 'Choose what to index: source files, pull requests, or both.',+ azure_devopsDescription: 'Connect Azure DevOps to sync repository files and pull requests.',bitbucketDescription: 'Connect Bitbucket to sync PR content.',bitbucketTopWorkspaceTip:'The Bitbucket workspace to index (e.g., "atlassian" from https://bitbucket.org/atlassian/workspace ).',diff --git a/web/src/locales/fr.ts b/web/src/locales/fr.tsindex 2f072734bd..5a480196cf 100644--- a/web/src/locales/fr.ts+++ b/web/src/locales/fr.ts@@ -1501,6 +1501,7 @@ Exemple : Virtual Hosted Style`,'Connectez un site SharePoint via Microsoft Graph pour synchroniser ses bibliothèques de documents.',sharepointSiteUrlTip:'URL complète du site SharePoint à indexer, ex. : https://contoso.sharepoint.com/sites/MonSite. Nécessite une application Azure AD avec les permissions applicatives Sites.Read.All et Files.Read.All (consentement administrateur).',+ azure_devopsDescription: 'Connectez Azure DevOps pour synchroniser les fichiers du dépôt et les pull requests.',bitbucketDescription:'Connectez Bitbucket pour synchroniser le contenu des PR.',bitbucketTopWorkspaceTip:diff --git a/web/src/locales/it.ts b/web/src/locales/it.tsindex e315686b99..512009e85d 100644--- a/web/src/locales/it.ts+++ b/web/src/locales/it.ts@@ -1280,6 +1280,7 @@ Esempio: Virtual Hosted Style`,'Connetti un sito SharePoint tramite Microsoft Graph per sincronizzare le sue librerie documenti.',sharepointSiteUrlTip:"URL completo del sito SharePoint da indicizzare, es. https://contoso.sharepoint.com/sites/MySite. Richiede un'app Azure AD con permessi applicativi Sites.Read.All e Files.Read.All (consenso admin).",+ azure_devopsDescription: 'Collega Azure DevOps per sincronizzare i file del repository e le pull request.',bitbucketDescription:'Connetti Bitbucket per sincronizzare il contenuto delle PR.',bitbucketTopWorkspaceTip:diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.tsindex 2e19ecc2de..ae6bee68e3 100644--- a/web/src/locales/ja.ts+++ b/web/src/locales/ja.ts@@ -1335,6 +1335,7 @@ export default {'Microsoft Graph経由でSharePointサイトに接続し、ドキュメントライブラリを同期します。',sharepointSiteUrlTip:'インデックス対象のSharePointサイトの完全なURLです(例: https://contoso.sharepoint.com/sites/MySite)。Sites.Read.AllおよびFiles.Read.Allのアプリケーション権限(管理者の同意)を持つAzure ADアプリが必要です。',+ azure_devopsDescription: 'Azure DevOps に接続し、リポジトリのファイルとプルリクエストを同期します。',bitbucketDescription: 'Bitbucketに接続し、PRの内容を同期します。',bitbucketTopWorkspaceTip:'インデックス対象のBitbucketワークスペースです(例: https://bitbucket.org/atlassian/workspace の「atlassian」)。',diff --git a/web/src/locales/ko.ts b/web/src/locales/ko.tsindex 3fd907fcce..6a3b13fa74 100644--- a/web/src/locales/ko.ts+++ b/web/src/locales/ko.ts@@ -1252,6 +1252,7 @@ export default {'Microsoft Graph를 통해 SharePoint 사이트를 연결하여 문서 라이브러리를 동기화합니다.',sharepointSiteUrlTip:'인덱싱할 SharePoint 사이트의 전체 URL (예: https://contoso.sharepoint.com/sites/MySite). Sites.Read.All 및 Files.Read.All 애플리케이션 권한이 있는 Azure AD 앱이 필요합니다 (관리자 동의).',+ azure_devopsDescription: 'Azure DevOps를 연결하여 리포지토리 파일과 풀 리퀘스트를 동기화합니다.',bitbucketDescription: 'Bitbucket을 연결하여 PR 콘텐츠를 동기화합니다.',bitbucketTopWorkspaceTip:'인덱싱할 Bitbucket 워크스페이스 (예: https://bitbucket.org/atlassian/workspace 의 "atlassian").',diff --git a/web/src/locales/ru.ts b/web/src/locales/ru.tsindex f0c615bedb..69e9089745 100644--- a/web/src/locales/ru.ts+++ b/web/src/locales/ru.ts@@ -1063,6 +1063,7 @@ export default {'Загрузите OAuth JSON, сгенерированный в Google Console. Если он содержит только учетные данные клиента, выполните одноразовое подтверждение в браузере, чтобы получить долгоживущие токены обновления.',dropboxDescription:'Подключите ваш Dropbox для синхронизации файлов и папок из выбранного аккаунта.',+ azure_devopsDescription: "Подключите Azure DevOps для синхронизации файлов репозитория и pull request'ов.",bitbucketDescription:'Подключите Bitbucket для синхронизации содержимого PR.',bitbucketTopWorkspaceTip:diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.tsindex c38a72d65a..7b92f22957 100644--- a/web/src/locales/tr.ts+++ b/web/src/locales/tr.ts@@ -1221,6 +1221,7 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman"Google Console'dan oluşturulan OAuth JSON'ını yükleyin. Yalnızca istemci kimlik bilgilerini içeriyorsa, uzun ömürlü yenileme tokenları oluşturmak için tarayıcı tabanlı doğrulamayı bir kez çalıştırın.",dropboxDescription:"Seçilen bir hesaptan dosya ve klasörleri senkronize etmek için Dropbox'ınızı bağlayın.",+ azure_devopsDescription: "Depo dosyalarını ve pull request'leri senkronize etmek için Azure DevOps'u bağlayın.",bitbucketDescription:"PR içeriğini senkronize etmek için Bitbucket'ı bağlayın.",bitbucketTopWorkspaceTip:diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.tsindex a77d2a5ce7..320df96a9a 100644--- a/web/src/locales/zh-traditional.ts+++ b/web/src/locales/zh-traditional.ts@@ -779,6 +779,7 @@ export default {modelsToBeAddedTooltip:'若您的模型供應商未列於此處,但宣稱與 OpenAI 相容,可透過選擇「OpenAI-API-compatible」卡片來設定相關模型。',dropboxDescription: '連接 Dropbox,同步指定帳號下的文件與文件夾。',+ azure_devopsDescription: '連接 Azure DevOps 以同步儲存庫檔案和拉取請求。',bitbucketDescription: '連接 Bitbucket,同步 PR 內容。',zendeskDescription: '連接 Zendesk,同步工單、文章及其他內容。',bitbucketTopWorkspaceTip:diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 7d4a7f361e..42e527da96 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -1258,6 +1258,13 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 RsharepointSiteUrlTip:'要索引的 SharePoint 站点完整 URL,例如 https://contoso.sharepoint.com/sites/MySite。需要具备 Sites.Read.All 与 Files.Read.All 应用权限(管理员同意)的 Azure AD 应用。',boxDescription: '连接你的 Box 云盘以同步文件和文件夹。',+ azureDevOpsPatTip: '需要具有 Code (Read) 权限的个人访问令牌。',+ azureDevOpsOrganizationTip: '组织名称(例如 contoso),或自托管 Azure DevOps Server 的集合地址(例如 https://tfs.contoso.com/DefaultCollection)。',+ azureDevOpsProjectsTip: '以逗号分隔的团队项目名称。例如:Project1,Project2',+ azureDevOpsRepositoriesTip: '以逗号分隔的仓库。可使用 project/repo 形式以区分同名仓库。',+ azureDevOpsOrganizationScopeTip: '将索引该组织中令牌可见的所有仓库。',+ azureDevOpsContentTypesTip: '选择要索引的内容:源文件、拉取请求,或两者。',+ azure_devopsDescription: '连接 Azure DevOps 以同步仓库文件和拉取请求。',bitbucketDescription: '连接 Bitbucket,同步 PR 内容。',bitbucketTopWorkspaceTip:'要索引的 Bitbucket 工作区(例如:https://bitbucket.org/atlassian/workspace 中的 "atlassian")',@@ -1386,6 +1393,14 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 RdataSourceFieldWikiBaseUrl: 'Wiki 基础 URL',dataSourceFieldIsCloud: '是否为云版本',dataSourceFieldIndexMode: '索引模式',+ dataSourceFieldAzureDevOpsPat: 'Azure DevOps 个人访问令牌',+ dataSourceFieldAzureDevOpsOrganization: 'Azure DevOps 组织',+ dataSourceFieldAzureDevOpsRepositories: '仓库',+ dataSourceFieldAzureDevOpsContentTypes: '内容类型',+ dataSourceOptionOrganization: '组织',+ dataSourceOptionCode: '代码',+ dataSourceOptionPullRequests: '拉取请求',+ dataSourceOptionBoth: '两者',dataSourceFieldPageId: '页面 ID',dataSourceFieldSpaceKey: '空间标识(Space Key)',dataSourceFieldIndexRecursively: '递归索引',diff --git a/web/src/pages/user-setting/data-source/constant/azure-devops-constant.tsx b/web/src/pages/user-setting/data-source/constant/azure-devops-constant.tsxnew file mode 100644index 0000000000..b06465a175--- /dev/null+++ b/web/src/pages/user-setting/data-source/constant/azure-devops-constant.tsx@@ -0,0 +1,111 @@+/*+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { FilterFormField, FormFieldType } from '@/components/dynamic-form';+import { TFunction } from 'i18next';++export const azureDevOpsConstant = (t: TFunction) => [+ {+ label: t('setting.dataSourceFieldAzureDevOpsPat'),+ name: 'config.credentials.azure_devops_pat',+ type: FormFieldType.Password,+ required: true,+ tooltip: t('setting.azureDevOpsPatTip'),+ },+ {+ label: t('setting.dataSourceFieldAzureDevOpsOrganization'),+ name: 'config.organization',+ type: FormFieldType.Text,+ required: true,+ tooltip: t('setting.azureDevOpsOrganizationTip'),+ },+ {+ label: t('setting.dataSourceFieldIndexMode'),+ name: 'config.index_mode',+ type: FormFieldType.Segmented,+ options: [+ {+ label: t('setting.dataSourceOptionOrganization'),+ value: 'organization',+ },+ { label: t('setting.dataSourceOptionProjects'), value: 'projects' },+ {+ label: t('setting.dataSourceOptionRepositories'),+ value: 'repositories',+ },+ ],+ },+ {+ label: t('setting.dataSourceFieldProjects'),+ name: 'config.projects',+ type: FormFieldType.Text,+ customValidate: (val: string, formValues: any) => {+ const index_mode = formValues?.config?.index_mode;+ if (!val?.trim() && index_mode === 'projects') {+ return t('setting.dataSourceValidationFieldRequired', {+ label: t('setting.dataSourceFieldProjects'),+ });+ }+ return true;+ },+ shouldRender: (formValues: any) =>+ formValues?.config?.index_mode === 'projects',+ tooltip: t('setting.azureDevOpsProjectsTip'),+ },+ {+ label: t('setting.dataSourceFieldAzureDevOpsRepositories'),+ name: 'config.repositories',+ type: FormFieldType.Text,+ customValidate: (val: string, formValues: any) => {+ const index_mode = formValues?.config?.index_mode;+ if (!val?.trim() && index_mode === 'repositories') {+ return t('setting.dataSourceValidationFieldRequired', {+ label: t('setting.dataSourceFieldAzureDevOpsRepositories'),+ });+ }+ return true;+ },+ shouldRender: (formValues: any) =>+ formValues?.config?.index_mode === 'repositories',+ tooltip: t('setting.azureDevOpsRepositoriesTip'),+ },+ {+ name: FilterFormField + '.tip',+ label: ' ',+ type: FormFieldType.Custom,+ shouldRender: (formValues: any) =>+ formValues?.config?.index_mode === 'organization',+ render: () => (+ <div className="text-sm text-text-secondary bg-bg-card border border-border-button rounded-md px-3 py-2">+ {t('setting.azureDevOpsOrganizationScopeTip')}+ </div>+ ),+ },+ {+ label: t('setting.dataSourceFieldAzureDevOpsContentTypes'),+ name: 'config.content_types',+ type: FormFieldType.Segmented,+ options: [+ { label: t('setting.dataSourceOptionCode'), value: 'code' },+ {+ label: t('setting.dataSourceOptionPullRequests'),+ value: 'pull_requests',+ },+ { label: t('setting.dataSourceOptionBoth'), value: 'both' },+ ],+ tooltip: t('setting.azureDevOpsContentTypesTip'),+ },+];diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex f0bf2560f5..7e4cbb0a01 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -25,6 +25,7 @@ import BoxTokenField from '../component/box-token-field';import GmailTokenField from '../component/gmail-token-field';import GoogleDriveTokenField from '../component/google-drive-token-field';import { IDataSourceInfoMap } from '../interface';+import { azureDevOpsConstant } from './azure-devops-constant';import { bitbucketConstant } from './bitbucket-constant';import { confluenceConstant } from './confluence-constant';import { jiraConstant } from './jira-constant';@@ -44,6 +45,7 @@ export enum DataSourceKey {BOX = 'box',DROPBOX = 'dropbox',BITBUCKET = 'bitbucket',+ AZURE_DEVOPS = 'azure_devops',GITLAB = 'gitlab',GITHUB = 'github',MOODLE = 'moodle',@@ -126,6 +128,9 @@ export const DataSourceFeatureVisibilityMap: Partial<[DataSourceKey.BITBUCKET]: {syncDeletedFiles: true,},+ [DataSourceKey.AZURE_DEVOPS]: {+ syncDeletedFiles: true,+ },[DataSourceKey.AIRTABLE]: {syncDeletedFiles: true,},@@ -333,6 +338,11 @@ export const generateDataSourceInfo = (t: TFunction) => {description: t(`setting.${DataSourceKey.BITBUCKET}Description`),icon: <SvgIcon name={'data-source/bitbucket'} width={38} />,},+ [DataSourceKey.AZURE_DEVOPS]: {+ name: 'Azure DevOps',+ description: t(`setting.${DataSourceKey.AZURE_DEVOPS}Description`),+ icon: <SvgIcon name={'data-source/azure-devops'} width={38} />,+ },[DataSourceKey.ZENDESK]: {name: 'Zendesk',description: t(`setting.${DataSourceKey.ZENDESK}Description`),@@ -1342,6 +1352,7 @@ const generateDataSourceFormFields = (t: TFunction) => ({},],[DataSourceKey.BITBUCKET]: bitbucketConstant(t),+ [DataSourceKey.AZURE_DEVOPS]: azureDevOpsConstant(t),[DataSourceKey.ZENDESK]: [{label: t('setting.dataSourceFieldZendeskDomain'),@@ -2281,6 +2292,20 @@ export const DataSourceFormDefaultValues = {},},},+ [DataSourceKey.AZURE_DEVOPS]: {+ name: '',+ source: DataSourceKey.AZURE_DEVOPS,+ config: {+ organization: '',+ index_mode: 'organization',+ projects: '',+ repositories: '',+ content_types: 'both',+ credentials: {+ azure_devops_pat: '',+ },+ },+ },[DataSourceKey.BITBUCKET]: {name: '',source: DataSourceKey.BITBUCKET,