Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(github_pull_requests): store typed pull request columns in github_pull_request (#7103)
dev-rbf0f5b4f5f7f6
diff --git a/.sqlx/query-1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf.json b/.sqlx/query-1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf.jsonnew file mode 100644index 0000000000..f960121345--- /dev/null+++ b/.sqlx/query-1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH taken AS (\n SELECT 1 FROM github_pull_request WHERE github_key = $2\n ),\n removed AS (\n DELETE FROM github_pull_request\n WHERE github_key = $1 AND EXISTS (SELECT 1 FROM taken)\n )\n UPDATE github_pull_request\n SET github_key = $2, updated_at = NOW()\n WHERE github_key = $1 AND NOT EXISTS (SELECT 1 FROM taken)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf"+}diff --git a/.sqlx/query-45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9.json b/.sqlx/query-45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9.jsonnew file mode 100644index 0000000000..a7e8cea936--- /dev/null+++ b/.sqlx/query-45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9.json@@ -0,0 +1,26 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO github_pull_request (\n github_key,\n repository_id,\n number,\n owner,\n repo,\n title,\n status,\n draft,\n author_github_user_id,\n author_login,\n requested_reviewer_github_user_ids,\n participant_github_user_ids,\n github_updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)\n ON CONFLICT (github_key) DO UPDATE SET\n repository_id = COALESCE(EXCLUDED.repository_id, github_pull_request.repository_id),\n number = EXCLUDED.number,\n owner = EXCLUDED.owner,\n repo = EXCLUDED.repo,\n title = EXCLUDED.title,\n status = EXCLUDED.status,\n draft = EXCLUDED.draft,\n author_github_user_id = EXCLUDED.author_github_user_id,\n author_login = EXCLUDED.author_login,\n requested_reviewer_github_user_ids = EXCLUDED.requested_reviewer_github_user_ids,\n participant_github_user_ids = EXCLUDED.participant_github_user_ids,\n github_updated_at = EXCLUDED.github_updated_at,\n updated_at = NOW()\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Int8",+ "Int8",+ "Text",+ "Text",+ "Text",+ "Text",+ "Bool",+ "Text",+ "Text",+ "TextArray",+ "TextArray",+ "Timestamptz"+ ]+ },+ "nullable": []+ },+ "hash": "45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9"+}diff --git a/.sqlx/query-58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6.json b/.sqlx/query-58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6.jsonnew file mode 100644index 0000000000..fa1eabf5f9--- /dev/null+++ b/.sqlx/query-58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT DISTINCT id AS \"id!\"\n FROM github_app_installation\n WHERE $1::text IS NULL OR id > $1\n ORDER BY id\n LIMIT $2\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id!",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Int8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6"+}diff --git a/.sqlx/query-a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65.json b/.sqlx/query-a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65.jsonnew file mode 100644index 0000000000..fc15767c03--- /dev/null+++ b/.sqlx/query-a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT github_key\n FROM github_pull_request\n WHERE repository_id = $1 AND number = $2\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "github_key",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Int8",+ "Int8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65"+}diff --git a/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json b/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.jsonnew file mode 100644index 0000000000..e8fb0d829f--- /dev/null+++ b/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT DISTINCT ON (fe.foreign_entity_id) fe.metadata AS \"metadata!: serde_json::Value\"\n FROM foreign_entity fe\n WHERE fe.foreign_entity_source = $1::text\n AND left(lower(fe.foreign_entity_id), length($2::text)) = lower($2::text)\n ORDER BY fe.foreign_entity_id, fe.updated_at DESC, fe.id DESC\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "metadata!: serde_json::Value",+ "type_info": "Jsonb"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22"+}diff --git a/Cargo.lock b/Cargo.lockindex 27092a8b99..d53eeacc4d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8142,11 +8142,14 @@ dependencies = [name = "github_pull_requests"version = "0.1.0"dependencies = [+ "anyhow","chrono","entity_access","foreign_entity",+ "macro_db_migrator","serde","serde_json",+ "sqlx","thiserror 2.0.18","tokio","tracing",diff --git a/crates/github/src/domain/models.rs b/crates/github/src/domain/models.rsindex 109e2ea759..1a97dae2bd 100644--- a/crates/github/src/domain/models.rs+++ b/crates/github/src/domain/models.rs@@ -24,7 +24,7 @@ pub use installation_state::{InstallationState, InstallationStateError, sign_installation_state, verify_installation_state,};pub use link::{GithubAccessToken, GithubExchangeTokenResponse, GithubLink, GithubUserInfo};-pub use repository::GithubRepository;+pub use repository::{GithubRepository, PullRequestIndexPage, PullRequestIndexRequest};pub use sync::{GithubAppInstallationSource, GithubAuthenticatedUser, GithubInstallationAccessToken,GithubInstallationSetupAction, GithubSetupAccessToken, GithubUserInstallation,diff --git a/crates/github/src/domain/models/repository.rs b/crates/github/src/domain/models/repository.rsindex 30a54d1d59..e5836765d3 100644--- a/crates/github/src/domain/models/repository.rs+++ b/crates/github/src/domain/models/repository.rs@@ -18,6 +18,10 @@ pub struct GithubRepository {pub default_branch: Option<String>,/// Whether the repository is private.pub private: bool,+ /// GitHub's numeric repository id, which survives renames and transfers. Zero when it was+ /// not recorded.+ #[serde(default)]+ pub id: u64,}impl GithubRepository {@@ -29,3 +33,31 @@ impl GithubRepository {format!("https://github.com/{}/{}", self.owner, self.name)}}++/// A request for the next page of installations whose stored pull requests to index.+#[derive(Debug, Clone, Default, serde::Deserialize)]+#[serde(rename_all = "camelCase")]+pub struct PullRequestIndexRequest {+ /// Resume after this installation id; absent to start from the first installation.+ #[serde(default)]+ pub after: Option<String>,+ /// How many installations to process in this call; absent for the default page size.+ #[serde(default)]+ pub limit: Option<u32>,+}++/// What one page of pull request indexing did.+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]+#[serde(rename_all = "camelCase")]+pub struct PullRequestIndexPage {+ /// Installations processed in this page, including those that failed.+ pub installations: u32,+ /// Repositories listed across the page's installations.+ pub repositories: u32,+ /// Pull request rows written.+ pub indexed_pull_requests: u64,+ /// Installations whose repositories GitHub would not list, such as suspended ones.+ pub failed_installation_ids: Vec<String>,+ /// Pass as `after` to continue; absent once every installation has been processed.+ pub next_after: Option<String>,+}diff --git a/crates/github/src/domain/ports/mod.rs b/crates/github/src/domain/ports/mod.rsindex c8a9ff1a1c..4746fd5347 100644--- a/crates/github/src/domain/ports/mod.rs+++ b/crates/github/src/domain/ports/mod.rs@@ -11,5 +11,6 @@ mod sync;pub use link::{Auth, GithubLinkService, GithubOauth, GithubRepo};#[cfg(feature = "sync")]pub use sync::{- GithubRepositoryClient, GithubSyncClient, GithubSyncRealtime, GithubSyncRepo, GithubSyncService,+ GithubInstallationLister, GithubPullRequestIndex, GithubRepositoryClient, GithubSyncClient,+ GithubSyncRealtime, GithubSyncRepo, GithubSyncService,};diff --git a/crates/github/src/domain/ports/sync.rs b/crates/github/src/domain/ports/sync.rsindex 6fa456a3c3..a68b0c22d1 100644--- a/crates/github/src/domain/ports/sync.rs+++ b/crates/github/src/domain/ports/sync.rs@@ -8,7 +8,8 @@ use crate::domain::models::{AppJwt, EnrichedGithubPullRequest, GithubAppInstallationSource, GithubAuthenticatedUser,GithubError, GithubInstallationAccessToken, GithubKey, GithubPullRequestDetails,GithubRepository, GithubSetupAccessToken, GithubUserInstallation, MacroTaskId,- ResolvedTeamTaskReference, TeamTaskReference, ValidatedGithubWebhookEvent,+ PullRequestIndexPage, PullRequestIndexRequest, ResolvedTeamTaskReference, TeamTaskReference,+ ValidatedGithubWebhookEvent,};/// Repository for accessing github sync data from the database.@@ -310,6 +311,29 @@ pub trait GithubSyncRealtime: Send + Sync + 'static {) -> impl Future<Output = Result<(), GithubError>> + Send;}+/// Pages through every GitHub App installation Macro has recorded.+pub trait GithubInstallationLister: Send + Sync + 'static {+ /// The error type returned by repository operations.+ type Err: Into<anyhow::Error> + Send + std::fmt::Debug;++ /// Up to `limit` distinct installation ids that sort after `after`, in ascending order.+ fn list_installation_ids(+ &self,+ after: Option<&str>,+ limit: u32,+ ) -> impl Future<Output = Result<Vec<String>, Self::Err>> + Send;+}++/// Writes typed rows for pull requests stored before rows existed, one page of installations at+/// a time.+pub trait GithubPullRequestIndex: Send + Sync + 'static {+ /// Process the next page of installations and report what changed.+ fn index_pull_requests(+ &self,+ request: PullRequestIndexRequest,+ ) -> impl Future<Output = Result<PullRequestIndexPage, GithubError>> + Send;+}+/// Lists repository metadata for an installation the caller is authorized to access.pub trait GithubRepositoryClient: Send + Sync + 'static {/// Fetch the installation's repositories using metadata-only access.diff --git a/crates/github/src/domain/service/link_test.rs b/crates/github/src/domain/service/link_test.rsindex e35d20e0d5..55d6d86f43 100644--- a/crates/github/src/domain/service/link_test.rs+++ b/crates/github/src/domain/service/link_test.rs@@ -8,7 +8,10 @@ use foreign_entity::domain::{},ports::{ForeignEntityListQuery, ForeignEntityService},};-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;+use github_pull_requests::domain::{+ models::GithubPullRequestRow, ports::GithubPullRequestRepository,+ service::GithubPullRequestServiceImpl,+};use macro_user_id::{lowercased::Lowercase,user_id::{MacroUserId, MacroUserIdStr},@@ -210,6 +213,7 @@ fn default_pull_request_details() -> GithubPullRequestDetails {GithubPullRequestDetails {title: "Add token validation".to_string(),state: "open".to_string(),+ repository_id: None,merged_at: None,additions: 12,deletions: 3,@@ -219,6 +223,9 @@ fn default_pull_request_details() -> GithubPullRequestDetails {comments: None,checks: None,participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}}@@ -597,9 +604,31 @@ type TestGithubLinkService = GithubLinkServiceImpl<StubGithubRepo,StubGithubOauth,StubAuth,- GithubPullRequestServiceImpl<StubForeignEntityService>,+ GithubPullRequestServiceImpl<StubForeignEntityService, NoPullRequestRows>,>;+struct NoPullRequestRows;++impl GithubPullRequestRepository for NoPullRequestRows {+ type Err = std::convert::Infallible;++ async fn github_key_for(+ &self,+ _repository_id: i64,+ _number: i64,+ ) -> Result<Option<String>, Self::Err> {+ Ok(None)+ }++ async fn upsert_row(&self, _row: &GithubPullRequestRow) -> Result<(), Self::Err> {+ Ok(())+ }++ async fn rename_row(&self, _from: &str, _to: &str) -> Result<(), Self::Err> {+ Ok(())+ }+}+fn service(repo: StubGithubRepo, oauth: StubGithubOauth, auth: StubAuth) -> TestGithubLinkService {service_with_foreign_entities(repo, oauth, auth, StubForeignEntityService::default())}@@ -614,7 +643,7 @@ fn service_with_foreign_entities(repo,oauth,auth,- GithubPullRequestServiceImpl::new(foreign_entity_service),+ GithubPullRequestServiceImpl::new(foreign_entity_service, NoPullRequestRows),GithubLinkConfig {client_id: "client-id".to_string(),client_secret: "client-secret".to_string(),diff --git a/crates/github/src/domain/service/mod.rs b/crates/github/src/domain/service/mod.rsindex a563118439..f8bda7f691 100644--- a/crates/github/src/domain/service/mod.rs+++ b/crates/github/src/domain/service/mod.rs@@ -3,6 +3,8 @@#[cfg(feature = "sync")]mod installation_tokens;#[cfg(feature = "sync")]+mod pull_request_index;+#[cfg(feature = "sync")]mod reachable_repositories;#[cfg(feature = "sync")]mod sync;@@ -10,6 +12,8 @@ mod sync;#[cfg(feature = "sync")]pub use installation_tokens::{InstallationTokenConfig, InstallationTokenService};#[cfg(feature = "sync")]+pub use pull_request_index::PullRequestIndexService;+#[cfg(feature = "sync")]pub use reachable_repositories::ReachableRepositoriesService;#[cfg(feature = "sync")]pub use sync::{GithubSyncConfig, GithubSyncServiceImpl};diff --git a/crates/github/src/domain/service/pull_request_index.rs b/crates/github/src/domain/service/pull_request_index.rsnew file mode 100644index 0000000000..ae2f2ab476--- /dev/null+++ b/crates/github/src/domain/service/pull_request_index.rs@@ -0,0 +1,160 @@+//! Writing typed rows for pull requests stored before rows existed.+//!+//! A pull request row is identified by its repository's numeric id, which older records do not+//! carry. This pages through every installation, asks GitHub which repositories it covers, and+//! has the pull request store write a row for each record synced under one of those+//! repositories' names. Rows are rewritten from the record each time, so a page can be re-run+//! safely.++use github_pull_requests::domain::{+ models::GithubRepositoryIdentity, ports::GithubPullRequestIndexer,+};++use crate::domain::models::{+ AppJwt, GithubError, PullRequestIndexPage, PullRequestIndexRequest, app_jwt,+};+use crate::domain::ports::{+ GithubInstallationLister, GithubPullRequestIndex, GithubRepositoryClient,+};++use super::InstallationTokenConfig;++#[cfg(test)]+mod test;++/// Installations processed per call when the request does not say.+const DEFAULT_PAGE_SIZE: u32 = 25;+/// The most installations one call processes, keeping a call well inside request timeouts.+const MAX_PAGE_SIZE: u32 = 100;++/// Indexes stored pull requests one page of installations at a time.+pub struct PullRequestIndexService<Installations, Client, PullRequests> {+ config: InstallationTokenConfig,+ installations: Installations,+ client: Client,+ pull_requests: PullRequests,+}++impl<Installations, Client, PullRequests>+ PullRequestIndexService<Installations, Client, PullRequests>+{+ /// Build the service over the App's credentials, the recorded installations, a GitHub+ /// client, and the pull request store.+ pub fn new(+ config: InstallationTokenConfig,+ installations: Installations,+ client: Client,+ pull_requests: PullRequests,+ ) -> Self {+ Self {+ config,+ installations,+ client,+ pull_requests,+ }+ }+}++impl<Installations, Client, PullRequests> GithubPullRequestIndex+ for PullRequestIndexService<Installations, Client, PullRequests>+where+ Installations: GithubInstallationLister,+ Client: GithubRepositoryClient,+ PullRequests: GithubPullRequestIndexer,+{+ #[tracing::instrument(skip(self), err)]+ async fn index_pull_requests(+ &self,+ request: PullRequestIndexRequest,+ ) -> Result<PullRequestIndexPage, GithubError> {+ let limit = request+ .limit+ .unwrap_or(DEFAULT_PAGE_SIZE)+ .clamp(1, MAX_PAGE_SIZE);+ let installation_ids = self+ .installations+ .list_installation_ids(request.after.as_deref(), limit)+ .await+ .map_err(|error| {+ GithubError::Internal(anyhow::anyhow!("could not list installations: {error:?}"))+ })?;++ // A full page may have more after it; a short one is the last.+ let next_after = if installation_ids.len() == limit as usize {+ installation_ids.last().cloned()+ } else {+ None+ };+ let mut page = PullRequestIndexPage {+ installations: 0,+ repositories: 0,+ indexed_pull_requests: 0,+ failed_installation_ids: Vec::new(),+ next_after,+ };+ if installation_ids.is_empty() {+ return Ok(page);+ }++ let jwt = app_jwt(&self.config.client_id, &self.config.private_key_pem)?;+ for installation_id in installation_ids {+ page.installations += 1;+ let Some(repositories) = self.list_repositories(&jwt, &installation_id).await else {+ page.failed_installation_ids.push(installation_id);+ continue;+ };+ page.repositories += u32::try_from(repositories.len()).unwrap_or(u32::MAX);+ page.indexed_pull_requests += self+ .pull_requests+ .index_repositories(&repositories)+ .await+ .map_err(|error| {+ GithubError::Internal(anyhow::anyhow!(+ "could not index pull requests for installation {installation_id}: {error}"+ ))+ })?;+ }++ Ok(page)+ }+}++impl<Installations, Client, PullRequests>+ PullRequestIndexService<Installations, Client, PullRequests>+where+ Client: GithubRepositoryClient,+{+ /// The installation's repositories, or `None` when GitHub will not list them, for example+ /// because the installation was suspended or removed.+ async fn list_repositories(+ &self,+ jwt: &AppJwt,+ installation_id: &str,+ ) -> Option<Vec<GithubRepositoryIdentity>> {+ let installation = installation_id+ .parse::<u64>()+ .inspect_err(|error| {+ tracing::warn!(error=?error, installation_id, "installation id is not a number");+ })+ .ok()?;+ let repositories = self+ .client+ .repositories_for_installation(jwt, installation)+ .await+ .inspect_err(|error| {+ tracing::warn!(error=?error, installation_id, "could not list installation repositories");+ })+ .ok()?;++ Some(+ repositories+ .into_iter()+ .map(|repository| GithubRepositoryIdentity {+ id: repository.id,+ owner: repository.owner,+ name: repository.name,+ })+ .collect(),+ )+ }+}diff --git a/crates/github/src/domain/service/pull_request_index/test.rs b/crates/github/src/domain/service/pull_request_index/test.rsnew file mode 100644index 0000000000..ac19880ad7--- /dev/null+++ b/crates/github/src/domain/service/pull_request_index/test.rs@@ -0,0 +1,258 @@+use std::collections::HashMap;+use std::sync::{Arc, Mutex};++use github_pull_requests::domain::models::GithubPullRequestError;++use super::*;+use crate::domain::models::GithubRepository;++const TEST_PEM: &str = include_str!("../installation_tokens/test_key.pem");++fn config() -> InstallationTokenConfig {+ InstallationTokenConfig {+ client_id: "Iv1.testclient".to_owned(),+ private_key_pem: TEST_PEM.to_owned(),+ }+}++fn repository(id: u64, owner: &str, name: &str) -> GithubRepository {+ GithubRepository {+ owner: owner.to_owned(),+ name: name.to_owned(),+ html_url: format!("https://github.com/{owner}/{name}"),+ default_branch: Some("main".to_owned()),+ private: true,+ id,+ }+}++fn identity(id: u64, owner: &str, name: &str) -> GithubRepositoryIdentity {+ GithubRepositoryIdentity {+ id,+ owner: owner.to_owned(),+ name: name.to_owned(),+ }+}++struct FakeInstallations(Vec<String>);++impl GithubInstallationLister for FakeInstallations {+ type Err = anyhow::Error;++ async fn list_installation_ids(+ &self,+ after: Option<&str>,+ limit: u32,+ ) -> Result<Vec<String>, Self::Err> {+ Ok(self+ .0+ .iter()+ .filter(|id| after.is_none_or(|after| id.as_str() > after))+ .take(limit as usize)+ .cloned()+ .collect())+ }+}++#[derive(Default)]+struct FakeClient {+ repositories: HashMap<u64, Vec<GithubRepository>>,+ unavailable: Vec<u64>,+}++impl GithubRepositoryClient for FakeClient {+ async fn repositories_for_installation(+ &self,+ _jwt: &AppJwt,+ installation_id: u64,+ ) -> Result<Vec<GithubRepository>, GithubError> {+ if self.unavailable.contains(&installation_id) {+ return Err(GithubError::Internal(anyhow::anyhow!(+ "installation suspended"+ )));+ }+ Ok(self+ .repositories+ .get(&installation_id)+ .cloned()+ .unwrap_or_default())+ }+}++/// Records every batch it is given and reports one row per repository.+#[derive(Clone, Default)]+struct RecordingPullRequests {+ batches: Arc<Mutex<Vec<Vec<GithubRepositoryIdentity>>>>,+ fail: bool,+}++impl RecordingPullRequests {+ fn batches(&self) -> Vec<Vec<GithubRepositoryIdentity>> {+ self.batches.lock().expect("lock").clone()+ }+}++impl GithubPullRequestIndexer for RecordingPullRequests {+ async fn index_repositories(+ &self,+ repositories: &[GithubRepositoryIdentity],+ ) -> Result<u64, GithubPullRequestError> {+ if self.fail {+ return Err(GithubPullRequestError::Repository(anyhow::anyhow!(+ "database unavailable"+ )));+ }+ self.batches+ .lock()+ .expect("lock")+ .push(repositories.to_vec());+ Ok(repositories.len() as u64)+ }+}++fn service(+ installations: &[&str],+ client: FakeClient,+ pull_requests: RecordingPullRequests,+) -> PullRequestIndexService<FakeInstallations, FakeClient, RecordingPullRequests> {+ PullRequestIndexService::new(+ config(),+ FakeInstallations(installations.iter().map(|id| id.to_string()).collect()),+ client,+ pull_requests,+ )+}++fn two_installation_client() -> FakeClient {+ FakeClient {+ repositories: HashMap::from([+ (+ 1,+ vec![+ repository(100, "macro-inc", "macro"),+ repository(200, "macro-inc", "docs"),+ ],+ ),+ (3, vec![repository(300, "other-org", "tools")]),+ ]),+ unavailable: Vec::new(),+ }+}++#[tokio::test]+async fn pages_through_installations_and_reports_where_to_resume() {+ let pull_requests = RecordingPullRequests::default();+ let service = service(+ &["1", "2", "3"],+ two_installation_client(),+ pull_requests.clone(),+ );++ let first = service+ .index_pull_requests(PullRequestIndexRequest {+ after: None,+ limit: Some(2),+ })+ .await+ .expect("first page should succeed");+ assert_eq!(+ first,+ PullRequestIndexPage {+ installations: 2,+ repositories: 2,+ indexed_pull_requests: 2,+ failed_installation_ids: Vec::new(),+ next_after: Some("2".to_string()),+ }+ );++ let second = service+ .index_pull_requests(PullRequestIndexRequest {+ after: first.next_after,+ limit: Some(2),+ })+ .await+ .expect("second page should succeed");+ assert_eq!(+ second,+ PullRequestIndexPage {+ installations: 1,+ repositories: 1,+ indexed_pull_requests: 1,+ failed_installation_ids: Vec::new(),+ next_after: None,+ }+ );++ assert_eq!(+ pull_requests.batches(),+ vec![+ vec![+ identity(100, "macro-inc", "macro"),+ identity(200, "macro-inc", "docs"),+ ],+ vec![],+ vec![identity(300, "other-org", "tools")],+ ]+ );+}++#[tokio::test]+async fn installations_github_will_not_list_are_reported_and_skipped() {+ let pull_requests = RecordingPullRequests::default();+ let client = FakeClient {+ unavailable: vec![1],+ ..two_installation_client()+ };+ let service = service(&["1", "3"], client, pull_requests.clone());++ let page = service+ .index_pull_requests(PullRequestIndexRequest::default())+ .await+ .expect("page should succeed");++ assert_eq!(page.failed_installation_ids, vec!["1".to_string()]);+ assert_eq!(page.installations, 2);+ assert_eq!(page.indexed_pull_requests, 1);+ assert_eq!(page.next_after, None);+ assert_eq!(+ pull_requests.batches(),+ vec![vec![identity(300, "other-org", "tools")]]+ );+}++#[tokio::test]+async fn a_storage_failure_stops_the_page() {+ let pull_requests = RecordingPullRequests {+ fail: true,+ ..RecordingPullRequests::default()+ };+ let service = service(&["1"], two_installation_client(), pull_requests);++ let error = service+ .index_pull_requests(PullRequestIndexRequest::default())+ .await+ .expect_err("a storage failure should fail the page");++ assert!(matches!(error, GithubError::Internal(_)));+}++#[tokio::test]+async fn a_zero_limit_still_processes_one_installation() {+ let service = service(+ &["1", "3"],+ two_installation_client(),+ RecordingPullRequests::default(),+ );++ let page = service+ .index_pull_requests(PullRequestIndexRequest {+ after: None,+ limit: Some(0),+ })+ .await+ .expect("page should succeed");++ assert_eq!(page.installations, 1);+ assert_eq!(page.next_after, Some("1".to_string()));+}diff --git a/crates/github/src/domain/service/reachable_repositories/test.rs b/crates/github/src/domain/service/reachable_repositories/test.rsindex e241ed86c8..bd9cd04b42 100644--- a/crates/github/src/domain/service/reachable_repositories/test.rs+++ b/crates/github/src/domain/service/reachable_repositories/test.rs@@ -30,6 +30,7 @@ fn repository(owner: &str, name: &str) -> GithubRepository {html_url: format!("https://github.com/{owner}/{name}"),default_branch: Some("main".to_owned()),private: true,+ id: 0,}}diff --git a/crates/github/src/domain/service/sync.rs b/crates/github/src/domain/service/sync.rsindex 4fdb79403d..b355400e1c 100644--- a/crates/github/src/domain/service/sync.rs+++ b/crates/github/src/domain/service/sync.rs@@ -230,6 +230,11 @@ impl<github_key: github_key.as_ref().to_string(),owner: owner.to_string(),repo: repo.to_string(),+ repository_id: event+ .payload+ .get("repository")+ .and_then(|repository| repository.get("id"))+ .and_then(|value| value.as_u64()),number,url,display_name: format!("{owner}/{repo}#{number}"),@@ -260,6 +265,24 @@ impl<comments: None,checks: None,participant_github_user_ids: Self::participant_ids_from_payload(pull_request),+ draft: pull_request+ .and_then(|pr| pr.get("draft"))+ .and_then(|value| value.as_bool()),+ requested_reviewer_github_user_ids: pull_request+ .and_then(|pr| pr.get("requested_reviewers"))+ .and_then(|value| value.as_array())+ .map(|users| {+ users+ .iter()+ .filter_map(|user| user.get("id").and_then(|value| value.as_u64()))+ .map(|id| id.to_string())+ .collect()+ }),+ github_updated_at: pull_request+ .and_then(|pr| pr.get("updated_at"))+ .and_then(|value| value.as_str())+ .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok())+ .map(|updated_at| updated_at.with_timezone(&chrono::Utc)),})}@@ -348,6 +371,7 @@ impl<github_key: fallback.github_key,owner: fallback.owner,repo: fallback.repo,+ repository_id: details.repository_id.or(fallback.repository_id),number: fallback.number,url: fallback.url,display_name: fallback.display_name,@@ -363,6 +387,11 @@ impl<participant_github_user_ids: details.participant_github_user_ids.or(fallback.participant_github_user_ids),+ draft: details.draft.or(fallback.draft),+ requested_reviewer_github_user_ids: details+ .requested_reviewer_github_user_ids+ .or(fallback.requested_reviewer_github_user_ids),+ github_updated_at: details.github_updated_at.or(fallback.github_updated_at),}}diff --git a/crates/github/src/domain/service/sync/test.rs b/crates/github/src/domain/service/sync/test.rsindex 87e074ebf5..c4fd19e1c0 100644--- a/crates/github/src/domain/service/sync/test.rs+++ b/crates/github/src/domain/service/sync/test.rs@@ -33,7 +33,10 @@ use foreign_entity::domain::{},ports::{ForeignEntityListQuery, ForeignEntityService},};-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;+use github_pull_requests::domain::{+ models::GithubPullRequestRow, ports::GithubPullRequestRepository,+ service::GithubPullRequestServiceImpl,+};use macro_user_id::user_id::MacroUserIdStr;use model::document::{DocumentBasic, DocumentMetadata};use model_entity::Entity;@@ -1177,7 +1180,7 @@ type TestSyncServiceImpl = GithubSyncServiceImpl<StubDocumentService,StubSyncRepo,StubSyncClient,- GithubPullRequestServiceImpl<StubForeignEntityService>,+ TestPullRequestService,StubNotificationIngress,StubRealtime,>;@@ -1204,11 +1207,37 @@ impl std::ops::DerefMut for TestGithubSyncService {type TestServiceWithForeignEntityService = (TestGithubSyncService, StubForeignEntityService);+type TestPullRequestService =+ GithubPullRequestServiceImpl<StubForeignEntityService, NoPullRequestRows>;++struct NoPullRequestRows;++impl GithubPullRequestRepository for NoPullRequestRows {+ type Err = std::convert::Infallible;++ async fn github_key_for(+ &self,+ _repository_id: i64,+ _number: i64,+ ) -> Result<Option<String>, Self::Err> {+ Ok(None)+ }++ async fn upsert_row(&self, _row: &GithubPullRequestRow) -> Result<(), Self::Err> {+ Ok(())+ }++ async fn rename_row(&self, _from: &str, _to: &str) -> Result<(), Self::Err> {+ Ok(())+ }+}+fn pull_request_service(foreign_entity_service: &StubForeignEntityService,-) -> Arc<GithubPullRequestServiceImpl<StubForeignEntityService>> {+) -> Arc<TestPullRequestService> {Arc::new(GithubPullRequestServiceImpl::new(foreign_entity_service.clone(),+ NoPullRequestRows,))}@@ -1303,6 +1332,7 @@ fn expected_pull_request_metadata(github_key: "my-org/my-repo/pull/42".to_string(),owner: "my-org".to_string(),repo: "my-repo".to_string(),+ repository_id: None,number: 42,url: "https://github.com/my-org/my-repo/pull/42".to_string(),display_name: "my-org/my-repo#42".to_string(),@@ -1316,6 +1346,9 @@ fn expected_pull_request_metadata(comments: None,checks: None,participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}).unwrap()}@@ -1480,6 +1513,7 @@ fn backfilled_pull_request(title: &str) -> EnrichedGithubPullRequest {github_key: "my-org/my-repo/pull/42".to_string(),owner: "my-org".to_string(),repo: "my-repo".to_string(),+ repository_id: None,number: 42,url: "https://github.com/my-org/my-repo/pull/42".to_string(),display_name: "my-org/my-repo#42".to_string(),@@ -1493,6 +1527,9 @@ fn backfilled_pull_request(title: &str) -> EnrichedGithubPullRequest {comments: None,checks: None,participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}}@@ -1503,6 +1540,7 @@ fn expected_pull_request_metadata_from_details(github_key: "my-org/my-repo/pull/42".to_string(),owner: "my-org".to_string(),repo: "my-repo".to_string(),+ repository_id: None,number: 42,url: "https://github.com/my-org/my-repo/pull/42".to_string(),display_name: "my-org/my-repo#42".to_string(),@@ -1516,6 +1554,9 @@ fn expected_pull_request_metadata_from_details(comments: details.comments.clone(),checks: details.checks.clone(),participant_github_user_ids: details.participant_github_user_ids.clone(),+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}).unwrap()}@@ -1563,6 +1604,7 @@ fn pull_request_details(GithubPullRequestDetails {title: title.to_string(),state: "open".to_string(),+ repository_id: None,merged_at: None,additions,deletions,@@ -1572,6 +1614,9 @@ fn pull_request_details(comments,checks,participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}}diff --git a/crates/github/src/inbound.rs b/crates/github/src/inbound.rsindex fbd008912d..d882de9aee 100644--- a/crates/github/src/inbound.rs+++ b/crates/github/src/inbound.rs@@ -2,6 +2,8 @@#[cfg(all(feature = "axum", feature = "sync"))]pub mod github_sync_router;+#[cfg(all(feature = "axum", feature = "sync"))]+pub mod pull_request_index_router;#[cfg(feature = "axum")]impl axum::response::IntoResponse for crate::domain::models::GithubError {diff --git a/crates/github/src/inbound/pull_request_index_router.rs b/crates/github/src/inbound/pull_request_index_router.rsnew file mode 100644index 0000000000..4e137b18a8--- /dev/null+++ b/crates/github/src/inbound/pull_request_index_router.rs@@ -0,0 +1,88 @@+//! Internal route that writes typed rows for stored GitHub pull requests, one page of+//! installations per call.+//!+//! Run it once per environment after deploying, then again whenever rows need rebuilding. Each+//! call is safe to repeat: rows are rewritten from their records.+//!+//! ```sh+//! after=null+//! while :; do+//! page=$(curl -sf -X POST "$DSS_URL/internal/github/index-pull-requests" \+//! -H "x-internal-auth-key: $INTERNAL_API_KEY" \+//! -H 'content-type: application/json' \+//! -d "{\"after\": $after}")+//! echo "$page"+//! after=$(echo "$page" | jq '.nextAfter')+//! [ "$after" = "null" ] && break+//! done+//! ```++use std::sync::Arc;++use axum::{+ Json, Router,+ extract::{FromRef, State},+ routing::post,+};+use macro_authorization::{+ InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,+};++use crate::domain::{+ models::{GithubError, PullRequestIndexPage, PullRequestIndexRequest},+ ports::GithubPullRequestIndex,+};++/// State for the pull request index route.+pub struct PullRequestIndexRouterState<S, Auth> {+ /// The index service.+ pub service: Arc<S>,+ /// Authorization state used to require an internal caller.+ pub authorization_state: MacroAuthorizationState<Auth>,+}++impl<S, Auth> Clone for PullRequestIndexRouterState<S, Auth> {+ fn clone(&self) -> Self {+ Self {+ service: self.service.clone(),+ authorization_state: self.authorization_state.clone(),+ }+ }+}++impl<S, Auth> FromRef<PullRequestIndexRouterState<S, Auth>> for MacroAuthorizationState<Auth> {+ fn from_ref(state: &PullRequestIndexRouterState<S, Auth>) -> Self {+ state.authorization_state.clone()+ }+}++/// Build the internal router exposing `POST /index-pull-requests`.+pub fn pull_request_index_router<S, Auth, T>(+ state: PullRequestIndexRouterState<S, Auth>,+) -> Router<T>+where+ S: GithubPullRequestIndex,+ Auth: MacroAuthorizationService,+ T: Send + Sync + 'static,+{+ Router::new()+ .route(+ "/index-pull-requests",+ post(index_pull_requests_handler::<S, Auth>),+ )+ .with_state(state)+}++/// Process the next page of installations.+#[tracing::instrument(skip_all, err)]+pub async fn index_pull_requests_handler<S, Auth>(+ State(state): State<PullRequestIndexRouterState<S, Auth>>,+ _internal: MacroAuthorizationExtractor<Auth, InternalOnly>,+ Json(request): Json<PullRequestIndexRequest>,+) -> Result<Json<PullRequestIndexPage>, GithubError>+where+ S: GithubPullRequestIndex,+ Auth: MacroAuthorizationService,+{+ Ok(Json(state.service.index_pull_requests(request).await?))+}diff --git a/crates/github/src/outbound/pg_github_sync_repo.rs b/crates/github/src/outbound/pg_github_sync_repo.rsindex 2434828cbe..1917c0f0bc 100644--- a/crates/github/src/outbound/pg_github_sync_repo.rs+++ b/crates/github/src/outbound/pg_github_sync_repo.rs@@ -13,7 +13,7 @@ use crate::domain::{GithubAppInstallationSource, GithubKey, MacroTaskId, ResolvedTeamTaskReference,TeamTaskReference,},- ports::GithubSyncRepo,+ ports::{GithubInstallationLister, GithubSyncRepo},};/// PostgreSQL-backed github repository.@@ -29,6 +29,31 @@ impl PgGithubSyncRepo {}}+impl GithubInstallationLister for PgGithubSyncRepo {+ type Err = sqlx::Error;++ #[tracing::instrument(skip(self), err)]+ async fn list_installation_ids(+ &self,+ after: Option<&str>,+ limit: u32,+ ) -> Result<Vec<String>, Self::Err> {+ sqlx::query_scalar!(+ r#"+ SELECT DISTINCT id AS "id!"+ FROM github_app_installation+ WHERE $1::text IS NULL OR id > $1+ ORDER BY id+ LIMIT $2+ "#,+ after,+ i64::from(limit),+ )+ .fetch_all(&self.pool)+ .await+ }+}+impl GithubSyncRepo for PgGithubSyncRepo {type Err = sqlx::Error;diff --git a/crates/github/src/outbound/pull_request_metadata.rs b/crates/github/src/outbound/pull_request_metadata.rsindex 64713d4616..f87c50df84 100644--- a/crates/github/src/outbound/pull_request_metadata.rs+++ b/crates/github/src/outbound/pull_request_metadata.rs@@ -50,8 +50,10 @@ pub(crate) async fn fetch_pull_request_metadata(.as_ref().and_then(|user| user.login.clone());let author_id = pull_request.user.as_ref().and_then(|user| user.id);+ let requested_reviewer_github_user_ids = user_ids(&pull_request.requested_reviewers);Ok(GithubPullRequestDetails {+ repository_id: pull_request.base.repository_id(),title: pull_request.title,state: pull_request.state,merged_at: pull_request.merged_at,@@ -63,6 +65,9 @@ pub(crate) async fn fetch_pull_request_metadata(comments,checks,participant_github_user_ids,+ draft: pull_request.draft,+ requested_reviewer_github_user_ids: Some(requested_reviewer_github_user_ids),+ github_updated_at: pull_request.updated_at,})}@@ -151,6 +156,7 @@ struct GithubInstallationRepositoriesResponse {#[derive(Debug, serde::Deserialize)]struct GithubInstallationRepositoryResponse {+ id: u64,name: String,owner: GithubRepositoryOwnerResponse,html_url: String,@@ -168,6 +174,7 @@ impl GithubInstallationRepositoryResponse {html_url: self.html_url,default_branch: self.default_branch,private: self.private,+ id: self.id,}}}@@ -183,6 +190,12 @@ struct GithubOpenPullRequestResponse {title: String,html_url: String,body: Option<String>,+ #[serde(default)]+ base: GithubPullRequestBaseResponse,+ #[serde(default)]+ draft: Option<bool>,+ #[serde(default)]+ updated_at: Option<chrono::DateTime<chrono::Utc>>,user: Option<GithubUserResponse>,#[serde(default)]requested_reviewers: Vec<GithubUserResponse>,@@ -199,6 +212,12 @@ struct GithubPullRequestResponse {deletions: u64,body: Option<String>,head: GithubPullRequestHeadResponse,+ #[serde(default)]+ base: GithubPullRequestBaseResponse,+ #[serde(default)]+ draft: Option<bool>,+ #[serde(default)]+ updated_at: Option<chrono::DateTime<chrono::Utc>>,user: Option<GithubUserResponse>,#[serde(default)]requested_reviewers: Vec<GithubUserResponse>,@@ -235,6 +254,31 @@ struct GithubPullRequestHeadResponse {sha: String,}+#[derive(Debug, Default, serde::Deserialize)]+struct GithubPullRequestBaseResponse {+ repo: Option<GithubRepositoryIdResponse>,+}++impl GithubPullRequestBaseResponse {+ fn repository_id(&self) -> Option<u64> {+ self.repo.as_ref().map(|repo| repo.id)+ }+}++#[derive(Debug, serde::Deserialize)]+struct GithubRepositoryIdResponse {+ id: u64,+}++/// The stable numeric ids, as strings, of the users that have one.+fn user_ids(users: &[GithubUserResponse]) -> Vec<String> {+ users+ .iter()+ .filter_map(|user| user.id)+ .map(|id| id.to_string())+ .collect()+}+#[derive(Debug, serde::Deserialize)]struct GithubUserResponse {login: Option<String>,@@ -597,6 +641,7 @@ impl GithubOpenPullRequestResponse {github_key: GithubKey::new(owner, repo, self.number).to_string(),owner: owner.to_string(),repo: repo.to_string(),+ repository_id: self.base.repository_id(),number: self.number,url: self.html_url,display_name: format!("{owner}/{repo}#{}", self.number),@@ -611,6 +656,9 @@ impl GithubOpenPullRequestResponse {checks: None,participant_github_user_ids: (!participant_ids.is_empty()).then(|| participant_ids.iter().map(u64::to_string).collect()),+ draft: self.draft,+ requested_reviewer_github_user_ids: Some(user_ids(&self.requested_reviewers)),+ github_updated_at: self.updated_at,}}}diff --git a/crates/github_pull_requests/Cargo.toml b/crates/github_pull_requests/Cargo.tomlindex 5798457bc1..22755bdde9 100644--- a/crates/github_pull_requests/Cargo.toml+++ b/crates/github_pull_requests/Cargo.toml@@ -5,22 +5,26 @@ publish = falseversion = "0.1.0"[features]-default = ["ports", "service"]+default = ["outbound", "ports", "service"]+outbound = ["dep:sqlx", "ports"]ports = []schema = ["dep:utoipa"]service = ["foreign_entity/ports", "ports"][dependencies]+anyhow = { workspace = true }chrono = { workspace = true }foreign_entity = { path = "../foreign_entity", default-features = false }serde = { workspace = true }serde_json = { workspace = true }+sqlx = { workspace = true, optional = true }thiserror = { workspace = true }tracing = { workspace = true }utoipa = { workspace = true, optional = true }+uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]entity_access = { path = "../entity_access", default-features = false, features = ["ports"] }+macro_db_migrator = { path = "../macro_db_migrator" }tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }-uuid = { workspace = true }diff --git a/crates/github_pull_requests/src/domain/models.rs b/crates/github_pull_requests/src/domain/models.rsindex 8ca24bf8cf..7a383c7dc0 100644--- a/crates/github_pull_requests/src/domain/models.rs+++ b/crates/github_pull_requests/src/domain/models.rs@@ -6,6 +6,7 @@ mod test;mod key;mod pull_request;+use chrono::{DateTime, Utc};use foreign_entity::domain::models::{ForeignEntity, ForeignEntityError, SourceId};pub use key::GithubKey;@@ -36,6 +37,80 @@ pub struct UpsertedGithubPullRequest {pub participant_github_user_ids: Vec<String>,}+/// A GitHub repository's stable id together with its current name.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct GithubRepositoryIdentity {+ /// The numeric GitHub repository id.+ pub id: u64,+ /// The account the repository lives under.+ pub owner: String,+ /// The repository name, without its owner.+ pub name: String,+}++/// A pull request's typed columns, read from its records' metadata.+#[derive(Debug, Clone, PartialEq)]+pub struct GithubPullRequestRow {+ /// The pull request's `owner/repo/pull/number` key.+ pub github_key: String,+ /// The numeric GitHub repository id. `None` when the metadata does not carry one, in which+ /// case an existing row keeps the id it has.+ pub repository_id: Option<i64>,+ /// The pull request number within its repository.+ pub number: i64,+ /// The repository owner the pull request was last synced under.+ pub owner: String,+ /// The repository name the pull request was last synced under.+ pub repo: String,+ /// The pull request title.+ pub title: Option<String>,+ /// The normalized pull request status.+ pub status: Option<GithubPullRequestStatus>,+ /// Whether the pull request is a draft.+ pub draft: bool,+ /// Stable numeric GitHub user id of the author.+ pub author_github_user_id: Option<String>,+ /// The author's GitHub login when the pull request was last synced.+ pub author_login: Option<String>,+ /// Stable numeric GitHub user ids of the users asked to review.+ pub requested_reviewer_github_user_ids: Vec<String>,+ /// Stable numeric GitHub user ids of everyone involved.+ pub participant_github_user_ids: Vec<String>,+ /// When GitHub last updated the pull request.+ pub github_updated_at: Option<DateTime<Utc>>,+}++impl GithubPullRequestRow {+ /// Read the typed columns from a pull request record's metadata. `None` when the metadata is+ /// not pull request metadata.+ pub fn from_metadata(metadata: &serde_json::Value) -> Option<Self> {+ let pull_request: EnrichedGithubPullRequest =+ serde_json::from_value(metadata.clone()).ok()?;++ Some(Self {+ repository_id: pull_request+ .repository_id+ .and_then(|id| i64::try_from(id).ok()),+ number: i64::try_from(pull_request.number).ok()?,+ github_key: pull_request.github_key,+ owner: pull_request.owner,+ repo: pull_request.repo,+ title: pull_request.name,+ status: pull_request.status,+ draft: pull_request.draft.unwrap_or(false),+ author_github_user_id: pull_request.author_id.map(|id| id.to_string()),+ author_login: pull_request.author_login,+ requested_reviewer_github_user_ids: pull_request+ .requested_reviewer_github_user_ids+ .unwrap_or_default(),+ participant_github_user_ids: pull_request+ .participant_github_user_ids+ .unwrap_or_default(),+ github_updated_at: pull_request.github_updated_at,+ })+ }+}+/// Errors from storing GitHub pull requests.#[derive(Debug, thiserror::Error)]pub enum GithubPullRequestError {@@ -45,4 +120,7 @@ pub enum GithubPullRequestError {/// The foreign entity store failed.#[error(transparent)]ForeignEntity(#[from] ForeignEntityError),+ /// The pull request row store failed.+ #[error("pull request row storage failed: {0}")]+ Repository(anyhow::Error),}diff --git a/crates/github_pull_requests/src/domain/models/pull_request.rs b/crates/github_pull_requests/src/domain/models/pull_request.rsindex e85966d27f..06ddd2af7a 100644--- a/crates/github_pull_requests/src/domain/models/pull_request.rs+++ b/crates/github_pull_requests/src/domain/models/pull_request.rs@@ -164,6 +164,10 @@ pub struct GithubPullRequestDetails {pub title: String,/// The raw GitHub pull request state, usually `open` or `closed`.pub state: String,+ /// The numeric id of the base repository, when available.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub repository_id: Option<u64>,/// The merge timestamp returned by GitHub, when the pull request was merged.pub merged_at: Option<chrono::DateTime<chrono::Utc>>,/// The number of added lines reported by GitHub.@@ -201,6 +205,22 @@ pub struct GithubPullRequestDetails {deserialize_with = "deserialize_optional_array")]pub participant_github_user_ids: Option<Vec<String>>,+ /// Whether the pull request is a draft, when available.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub draft: Option<bool>,+ /// Stable numeric GitHub user ids (as strings) of the users asked to review, when available.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(+ default,+ skip_serializing_if = "Option::is_none",+ deserialize_with = "deserialize_optional_array"+ )]+ pub requested_reviewer_github_user_ids: Option<Vec<String>>,+ /// When GitHub last updated the pull request, when available.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub github_updated_at: Option<chrono::DateTime<chrono::Utc>>,}impl GithubPullRequestDetails {@@ -221,6 +241,11 @@ pub struct EnrichedGithubPullRequest {pub owner: String,/// The GitHub repository name.pub repo: String,+ /// The numeric GitHub repository id, when known. Unlike `owner` and `repo`, it survives+ /// renames and transfers.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub repository_id: Option<u64>,/// The GitHub pull request number.pub number: u64,/// The public GitHub URL for the pull request.@@ -255,6 +280,18 @@ pub struct EnrichedGithubPullRequest {/// union rather than replacing it (partial write paths must not drop known participants).#[serde(default, skip_serializing_if = "Option::is_none")]pub participant_github_user_ids: Option<Vec<String>>,+ /// Whether the pull request is a draft, when known.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub draft: Option<bool>,+ /// Stable numeric GitHub user ids (as strings) of the users asked to review, when known.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub requested_reviewer_github_user_ids: Option<Vec<String>>,+ /// When GitHub last updated the pull request, when known.+ #[cfg_attr(feature = "schema", schema(ignore))]+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub github_updated_at: Option<chrono::DateTime<chrono::Utc>>,}impl EnrichedGithubPullRequest {@@ -264,6 +301,7 @@ impl EnrichedGithubPullRequest {github_key: reference.github_key,owner: reference.owner,repo: reference.repo,+ repository_id: None,number: reference.number,url: reference.url,display_name: reference.display_name,@@ -277,6 +315,9 @@ impl EnrichedGithubPullRequest {comments: None,checks: None,participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}}@@ -291,6 +332,7 @@ impl EnrichedGithubPullRequest {github_key: reference.github_key,owner: reference.owner,repo: reference.repo,+ repository_id: details.repository_id,number: reference.number,url: reference.url,display_name: reference.display_name,@@ -304,6 +346,9 @@ impl EnrichedGithubPullRequest {comments: details.comments,checks: details.checks,participant_github_user_ids: details.participant_github_user_ids,+ draft: details.draft,+ requested_reviewer_github_user_ids: details.requested_reviewer_github_user_ids,+ github_updated_at: details.github_updated_at,}}@@ -311,8 +356,9 @@ impl EnrichedGithubPullRequest {////// Partial refreshes may omit `comments` or `checks`. When an omitted field exists as an array in/// `existing_metadata`, the existing array is copied forward so richer metadata is not discarded.- /// The same applies to the scalar `authorLogin`, `authorId`, and `description` fields, which- /// fallback write paths (such as comment webhooks without a `pull_request` payload) omit.+ /// The same applies to `authorLogin`, `authorId`, `description`, `repositoryId`, `draft`,+ /// `requestedReviewerGithubUserIds`, and `githubUpdatedAt`, which fallback write paths (such+ /// as comment webhooks without a `pull_request` payload) omit.pub fn foreign_entity_metadata(&self,existing_metadata: Option<&serde_json::Value>,@@ -337,7 +383,15 @@ impl EnrichedGithubPullRequest {}}- for field in ["authorLogin", "authorId", "description"] {+ for field in [+ "authorLogin",+ "authorId",+ "description",+ "repositoryId",+ "draft",+ "requestedReviewerGithubUserIds",+ "githubUpdatedAt",+ ] {if metadata_object.contains_key(field) {continue;}diff --git a/crates/github_pull_requests/src/domain/models/test.rs b/crates/github_pull_requests/src/domain/models/test.rsindex e7be9e3d45..447dc19f1d 100644--- a/crates/github_pull_requests/src/domain/models/test.rs+++ b/crates/github_pull_requests/src/domain/models/test.rs@@ -25,6 +25,7 @@ fn pull_request_details(GithubPullRequestDetails {title: "Add pull request enrichment".to_string(),state: state.to_string(),+ repository_id: None,merged_at,additions: 42,deletions: 12,@@ -34,6 +35,9 @@ fn pull_request_details(comments: None,checks: None,participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}}@@ -152,6 +156,7 @@ fn pull_request_response_serializes_with_camel_case_fields() {github_key: reference.github_key,owner: reference.owner,repo: reference.repo,+ repository_id: None,number: reference.number,url: reference.url,display_name: reference.display_name,@@ -165,6 +170,9 @@ fn pull_request_response_serializes_with_camel_case_fields() {comments: Some(vec![pull_request_comment()]),checks: Some(vec![pull_request_check_run()]),participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}],};@@ -285,6 +293,7 @@ fn pull_request_enrichment_copies_details_fields() {let details = GithubPullRequestDetails {title: "Add pull request enrichment".to_string(),state: "closed".to_string(),+ repository_id: None,merged_at: Some(utc_datetime("2026-05-25T18:54:21Z")),additions: 42,deletions: 12,@@ -294,6 +303,9 @@ fn pull_request_enrichment_copies_details_fields() {comments: Some(comments.clone()),checks: Some(checks.clone()),participant_github_user_ids: Some(vec!["42".to_string(), "583231".to_string()]),+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,};let enriched = EnrichedGithubPullRequest::from_details(reference.clone(), details);@@ -336,6 +348,7 @@ fn pull_request_foreign_entity_metadata_serializes_enriched_pull_request() {let details = GithubPullRequestDetails {title: "Add pull request enrichment".to_string(),state: "closed".to_string(),+ repository_id: None,merged_at: Some(utc_datetime("2026-05-25T18:54:21Z")),additions: 42,deletions: 12,@@ -345,6 +358,9 @@ fn pull_request_foreign_entity_metadata_serializes_enriched_pull_request() {comments: Some(vec![pull_request_comment()]),checks: Some(vec![pull_request_check_run()]),participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,};let enriched = EnrichedGithubPullRequest::from_details(reference, details);@@ -457,6 +473,7 @@ fn pull_request_foreign_entity_metadata_keeps_fresh_arrays() {let details = GithubPullRequestDetails {title: "Add pull request enrichment".to_string(),state: "open".to_string(),+ repository_id: None,merged_at: None,additions: 42,deletions: 12,@@ -466,6 +483,9 @@ fn pull_request_foreign_entity_metadata_keeps_fresh_arrays() {comments: Some(comments.clone()),checks: Some(checks.clone()),participant_github_user_ids: None,+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,};let enriched = EnrichedGithubPullRequest::from_details(pull_request_reference(), details);let existing_metadata = serde_json::json!({diff --git a/crates/github_pull_requests/src/domain/ports.rs b/crates/github_pull_requests/src/domain/ports.rsindex 28a1e25ad1..4db7f2ada2 100644--- a/crates/github_pull_requests/src/domain/ports.rs+++ b/crates/github_pull_requests/src/domain/ports.rs@@ -5,8 +5,8 @@ use std::future::Future;use foreign_entity::domain::models::ForeignEntity;use super::models::{- EnrichedGithubPullRequest, GithubPullRequestError, UpsertGithubPullRequest,- UpsertedGithubPullRequest,+ EnrichedGithubPullRequest, GithubPullRequestError, GithubPullRequestRow,+ GithubRepositoryIdentity, UpsertGithubPullRequest, UpsertedGithubPullRequest,};/// Stores GitHub pull requests as foreign entity records, one record per user or team the pull@@ -14,6 +14,9 @@ use super::models::{pub trait GithubPullRequestService: Send + Sync + 'static {/// Store a pull request's latest data on the record for/// [`UpsertGithubPullRequest::stored_for`], creating it when that source has none.+ ///+ /// A pull request whose repository was renamed or transferred keeps its records: they are+ /// found by repository id and number and moved to the new key.fn upsert_pull_request(&self,upsert: UpsertGithubPullRequest,@@ -29,3 +32,55 @@ pub trait GithubPullRequestService: Send + Sync + 'static {pull_request: &EnrichedGithubPullRequest,) -> impl Future<Output = Result<Vec<ForeignEntity>, GithubPullRequestError>> + Send;}++/// Writes typed rows for pull requests stored before rows existed.+pub trait GithubPullRequestIndexer: Send + Sync + 'static {+ /// Write rows for the stored pull requests of each of `repositories`, matched by owner and+ /// name. Returns how many rows were written.+ fn index_repositories(+ &self,+ repositories: &[GithubRepositoryIdentity],+ ) -> impl Future<Output = Result<u64, GithubPullRequestError>> + Send;+}++/// Persists the typed columns of pull requests.+pub trait GithubPullRequestRepository: Send + Sync + 'static {+ /// Error type returned by repository operations.+ type Err: Into<anyhow::Error> + Send + std::fmt::Debug;++ /// The key of pull request `number` in the repository with id `repository_id`.+ fn github_key_for(+ &self,+ repository_id: i64,+ number: i64,+ ) -> impl Future<Output = Result<Option<String>, Self::Err>> + Send;++ /// Store a pull request's typed columns. A row without a repository id keeps the one it+ /// has.+ fn upsert_row(+ &self,+ row: &GithubPullRequestRow,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;++ /// Move the row stored under `from` to `to`. When `to` already has a row, the row under+ /// `from` is removed instead.+ fn rename_row(+ &self,+ from: &str,+ to: &str,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;+}++/// Reads the stored pull request records the indexer writes rows from.+pub trait GithubPullRequestIndexRepository: Send + Sync + 'static {+ /// Error type returned by repository operations.+ type Err: Into<anyhow::Error> + Send + std::fmt::Debug;++ /// The metadata of the most recently updated record of each pull request stored under the+ /// repository `owner/name`, compared case-insensitively.+ fn latest_pull_request_metadata(+ &self,+ owner: &str,+ name: &str,+ ) -> impl Future<Output = Result<Vec<serde_json::Value>, Self::Err>> + Send;+}diff --git a/crates/github_pull_requests/src/domain/service.rs b/crates/github_pull_requests/src/domain/service.rsindex 1070b7c710..4f2f454f95 100644--- a/crates/github_pull_requests/src/domain/service.rs+++ b/crates/github_pull_requests/src/domain/service.rs@@ -11,22 +11,28 @@ use foreign_entity::domain::{use super::{models::{EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,- GithubPullRequestError, GithubPullRequestStatus, UpsertGithubPullRequest,- UpsertedGithubPullRequest,+ GithubPullRequestError, GithubPullRequestRow, GithubPullRequestStatus,+ GithubRepositoryIdentity, UpsertGithubPullRequest, UpsertedGithubPullRequest,+ },+ ports::{+ GithubPullRequestIndexRepository, GithubPullRequestIndexer, GithubPullRequestRepository,+ GithubPullRequestService,},- ports::GithubPullRequestService,};-/// Stores pull requests through the foreign entity service.-pub struct GithubPullRequestServiceImpl<F> {+/// Stores pull requests through the foreign entity service, with their typed columns in `repo`.+pub struct GithubPullRequestServiceImpl<F, R> {foreign_entity_service: F,+ repo: R,}-impl<F: ForeignEntityService> GithubPullRequestServiceImpl<F> {- /// Create a service that stores pull requests through `foreign_entity_service`.- pub fn new(foreign_entity_service: F) -> Self {+impl<F: ForeignEntityService, R: GithubPullRequestRepository> GithubPullRequestServiceImpl<F, R> {+ /// Create a service that stores pull requests through `foreign_entity_service` and their+ /// typed columns in `repo`.+ pub fn new(foreign_entity_service: F, repo: R) -> Self {Self {foreign_entity_service,+ repo,}}@@ -60,9 +66,77 @@ impl<F: ForeignEntityService> GithubPullRequestServiceImpl<F> {).await?)}+ /// Move a pull request stored under the name its repository had before a rename or+ /// transfer to its current key: every record stored for it, and its row.+ async fn follow_rename(+ &self,+ pull_request: &EnrichedGithubPullRequest,+ ) -> Result<(), GithubPullRequestError> {+ let (Some(repository_id), Ok(number)) = (+ pull_request+ .repository_id+ .and_then(|id| i64::try_from(id).ok()),+ i64::try_from(pull_request.number),+ ) else {+ return Ok(());+ };+ let Some(previous_key) = self+ .repo+ .github_key_for(repository_id, number)+ .await+ .map_err(repository_error)?+ else {+ return Ok(());+ };+ if previous_key == pull_request.github_key {+ return Ok(());+ }++ for record in self.stored_records(&previous_key).await? {+ self.foreign_entity_service+ .patch_foreign_entity(+ record.id,+ PatchForeignEntity {+ foreign_entity_id: Some(pull_request.github_key.clone()),+ ..PatchForeignEntity::default()+ },+ )+ .await?;+ }+ self.repo+ .rename_row(&previous_key, &pull_request.github_key)+ .await+ .map_err(repository_error)+ }++ /// Write a pull request's typed columns from a record's metadata. Rows are derived data, so+ /// a failure is logged rather than failing the write of the record itself.+ async fn store_row(&self, metadata: &serde_json::Value, repository_id: Option<i64>) -> bool {+ let Some(mut row) = GithubPullRequestRow::from_metadata(metadata) else {+ tracing::warn!("pull request metadata has no typed columns");+ return false;+ };+ if repository_id.is_some() {+ row.repository_id = repository_id;+ }++ match self.repo.upsert_row(&row).await {+ Ok(()) => true,+ Err(error) => {+ tracing::error!(+ error=?error,+ github_key=%row.github_key,+ "failed to store pull request row"+ );+ false+ }+ }+ }}-impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServiceImpl<F> {+impl<F: ForeignEntityService, R: GithubPullRequestRepository> GithubPullRequestService+ for GithubPullRequestServiceImpl<F, R>+{#[tracing::instrument(err,skip(self, upsert),@@ -76,6 +150,7 @@ impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServpull_request,stored_for,} = upsert;+ self.follow_rename(&pull_request).await?;let records = self.stored_records(&pull_request.github_key).await?;let existing = records.iter().find(|record| {@@ -115,6 +190,7 @@ impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServ.await?}};+ self.store_row(&foreign_entity.metadata, None).await;Ok(UpsertedGithubPullRequest {foreign_entity,@@ -147,13 +223,56 @@ impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServ}}}- match first_error {- Some(error) => Err(error),- None => Ok(refreshed),+ // A successful subset can omit metadata held only by a failed record. Preserve the+ // existing typed row rather than replacing it with that incomplete projection.+ if let Some(error) = first_error {+ return Err(error);+ }+ if let Some(latest) = refreshed.iter().max_by_key(|record| record.updated_at) {+ self.store_row(&latest.metadata, None).await;+ }+ Ok(refreshed)+ }+}++impl<F, R> GithubPullRequestIndexer for GithubPullRequestServiceImpl<F, R>+where+ F: ForeignEntityService,+ R: GithubPullRequestRepository + GithubPullRequestIndexRepository,+{+ #[tracing::instrument(err, skip(self, repositories), fields(repositories = repositories.len()))]+ async fn index_repositories(+ &self,+ repositories: &[GithubRepositoryIdentity],+ ) -> Result<u64, GithubPullRequestError> {+ let mut indexed = 0;+ for repository in repositories {+ let Some(repository_id) = i64::try_from(repository.id).ok().filter(|id| *id != 0)+ else {+ continue;+ };+ let pull_requests = GithubPullRequestIndexRepository::latest_pull_request_metadata(+ &self.repo,+ &repository.owner,+ &repository.name,+ )+ .await+ .map_err(repository_error)?;+ for metadata in &pull_requests {+ if self.store_row(metadata, Some(repository_id)).await {+ indexed += 1;+ }+ }}++ Ok(indexed)}}+fn repository_error(error: impl Into<anyhow::Error>) -> GithubPullRequestError {+ GithubPullRequestError::Repository(error.into())+}+fn status_from_metadata(metadata: &serde_json::Value) -> Option<GithubPullRequestStatus> {metadata.get("status")diff --git a/crates/github_pull_requests/src/domain/service/test.rs b/crates/github_pull_requests/src/domain/service/test.rsindex bdbb540a7e..677b9b5e4a 100644--- a/crates/github_pull_requests/src/domain/service/test.rs+++ b/crates/github_pull_requests/src/domain/service/test.rs@@ -1,3 +1,4 @@+use std::convert::Infallible;use std::sync::{Arc, Mutex};use chrono::Utc;@@ -13,10 +14,13 @@ use uuid::Uuid;use super::GithubPullRequestServiceImpl;use crate::domain::{models::{- EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,- GithubPullRequestStatus, UpsertGithubPullRequest,+ EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, GithubPullRequestRow,+ GithubPullRequestStatus, GithubRepositoryIdentity, UpsertGithubPullRequest,+ },+ ports::{+ GithubPullRequestIndexRepository, GithubPullRequestIndexer, GithubPullRequestRepository,+ GithubPullRequestService,},- ports::GithubPullRequestService,};const GITHUB_KEY: &str = "macro/app/pull/7";@@ -140,6 +144,9 @@ impl ForeignEntityService for StubForeignEntityService {.iter_mut().find(|record| record.id == id).ok_or(ForeignEntityError::NotFound(id))?;+ if let Some(foreign_entity_id) = patch.foreign_entity_id {+ record.foreign_entity_id = foreign_entity_id;+ }if let Some(metadata) = patch.metadata {record.metadata = metadata;}@@ -148,11 +155,94 @@ impl ForeignEntityService for StubForeignEntityService {}}+#[derive(Clone, Default)]+struct StubPullRequestRows {+ rows: Arc<Mutex<Vec<GithubPullRequestRow>>>,+ /// Stored pull request metadata by repository owner and name.+ stored: Arc<Mutex<Vec<(String, String, serde_json::Value)>>>,+}++impl StubPullRequestRows {+ fn rows(&self) -> Vec<GithubPullRequestRow> {+ self.rows.lock().unwrap().clone()+ }+}++impl GithubPullRequestRepository for StubPullRequestRows {+ type Err = Infallible;++ async fn github_key_for(+ &self,+ repository_id: i64,+ number: i64,+ ) -> Result<Option<String>, Self::Err> {+ Ok(self+ .rows()+ .into_iter()+ .find(|row| row.repository_id == Some(repository_id) && row.number == number)+ .map(|row| row.github_key))+ }++ async fn upsert_row(&self, row: &GithubPullRequestRow) -> Result<(), Self::Err> {+ let mut rows = self.rows.lock().unwrap();+ match rows+ .iter_mut()+ .find(|existing| existing.github_key == row.github_key)+ {+ Some(existing) => {+ let repository_id = row.repository_id.or(existing.repository_id);+ *existing = GithubPullRequestRow {+ repository_id,+ ..row.clone()+ };+ }+ None => rows.push(row.clone()),+ }+ Ok(())+ }++ async fn rename_row(&self, from: &str, to: &str) -> Result<(), Self::Err> {+ let mut rows = self.rows.lock().unwrap();+ if rows.iter().any(|row| row.github_key == to) {+ rows.retain(|row| row.github_key != from);+ } else if let Some(row) = rows.iter_mut().find(|row| row.github_key == from) {+ row.github_key = to.to_owned();+ }+ Ok(())+ }+}++impl GithubPullRequestIndexRepository for StubPullRequestRows {+ type Err = Infallible;++ async fn latest_pull_request_metadata(+ &self,+ owner: &str,+ name: &str,+ ) -> Result<Vec<serde_json::Value>, Self::Err> {+ Ok(self+ .stored+ .lock()+ .unwrap()+ .iter()+ .filter(|(stored_owner, stored_name, _)| stored_owner == owner && stored_name == name)+ .map(|(_, _, metadata)| metadata.clone())+ .collect())+ }+}++type TestService = GithubPullRequestServiceImpl<StubForeignEntityService, StubPullRequestRows>;++fn service(foreign_entities: &StubForeignEntityService, rows: &StubPullRequestRows) -> TestService {+ GithubPullRequestServiceImpl::new(foreign_entities.clone(), rows.clone())+}+fn pull_request(status: GithubPullRequestStatus) -> EnrichedGithubPullRequest {EnrichedGithubPullRequest {github_key: GITHUB_KEY.to_string(),owner: "macro".to_string(),repo: "app".to_string(),+ repository_id: None,number: 7,url: "https://github.com/macro/app/pull/7".to_string(),display_name: "macro/app#7".to_string(),@@ -166,6 +256,9 @@ fn pull_request(status: GithubPullRequestStatus) -> EnrichedGithubPullRequest {comments: None,checks: None,participant_github_user_ids: Some(vec!["42".to_string()]),+ draft: None,+ requested_reviewer_github_user_ids: None,+ github_updated_at: None,}}@@ -194,7 +287,8 @@ fn user() -> SourceId {#[tokio::test]async fn upsert_creates_the_record_for_a_source_without_one() {let foreign_entities = StubForeignEntityService::default();- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);let upserted = service.upsert_pull_request(UpsertGithubPullRequest {@@ -231,7 +325,8 @@ async fn upsert_merges_into_the_sources_own_record() {let user_record = stored_record(&user(), serde_json::json!({ "status": "closed" }));let foreign_entities =StubForeignEntityService::with_records(vec![user_record.clone(), team_record.clone()]);- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);let upserted = service.upsert_pull_request(UpsertGithubPullRequest {@@ -271,7 +366,8 @@ async fn a_sources_first_record_starts_from_another_sources_metadata() {}),);let foreign_entities = StubForeignEntityService::with_records(vec![team_record]);- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);let upserted = service.upsert_pull_request(UpsertGithubPullRequest {@@ -296,7 +392,8 @@ async fn refresh_updates_every_stored_record() {let user_record = stored_record(&user(), serde_json::json!({ "status": "open" }));let foreign_entities =StubForeignEntityService::with_records(vec![team_record.clone(), user_record.clone()]);- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);let refreshed = service.refresh_pull_request(&pull_request(GithubPullRequestStatus::Closed))@@ -318,7 +415,8 @@ async fn refresh_updates_every_stored_record() {#[tokio::test]async fn refresh_without_stored_records_does_nothing() {let foreign_entities = StubForeignEntityService::default();- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);let refreshed = service.refresh_pull_request(&pull_request(GithubPullRequestStatus::Open))@@ -336,7 +434,13 @@ async fn refresh_attempts_later_records_after_a_patch_failure() {let foreign_entities =StubForeignEntityService::with_records(vec![team_record.clone(), user_record.clone()]);foreign_entities.state.lock().unwrap().failing_patches = vec![team_record.id];- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let original_metadata = pull_request(GithubPullRequestStatus::Open)+ .foreign_entity_metadata(None)+ .unwrap();+ let original_row = GithubPullRequestRow::from_metadata(&original_metadata).unwrap();+ rows.rows.lock().unwrap().push(original_row.clone());+ let service = service(&foreign_entities, &rows);let result = service.refresh_pull_request(&pull_request(GithubPullRequestStatus::Closed))@@ -360,6 +464,7 @@ async fn refresh_attempts_later_records_after_a_patch_failure() {assert_eq!(records[0].metadata["status"], "open");assert_eq!(records[1].metadata["status"], "closed");assert!(foreign_entities.creates().is_empty());+ assert_eq!(rows.rows(), vec![original_row]);}#[tokio::test]@@ -371,7 +476,8 @@ async fn refresh_attempts_every_record_when_all_patches_fail() {let ids = records.iter().map(|record| record.id).collect::<Vec<_>>();let foreign_entities = StubForeignEntityService::with_records(records.clone());foreign_entities.state.lock().unwrap().failing_patches = ids.clone();- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);assert!(service@@ -388,4 +494,148 @@ async fn refresh_attempts_every_record_when_all_patches_fail() {ids);assert_eq!(foreign_entities.records(), records);+ assert!(rows.rows().is_empty());+}++#[tokio::test]+async fn upsert_writes_the_row_from_the_merged_metadata() {+ let foreign_entities = StubForeignEntityService::default();+ let rows = StubPullRequestRows::default();+ let service = service(&foreign_entities, &rows);++ service+ .upsert_pull_request(UpsertGithubPullRequest {+ pull_request: EnrichedGithubPullRequest {+ repository_id: Some(99),+ draft: Some(true),+ requested_reviewer_github_user_ids: Some(vec!["8".to_string()]),+ ..pull_request(GithubPullRequestStatus::Open)+ },+ stored_for: user(),+ })+ .await+ .unwrap();++ assert_eq!(+ rows.rows(),+ vec![GithubPullRequestRow {+ github_key: GITHUB_KEY.to_string(),+ repository_id: Some(99),+ number: 7,+ owner: "macro".to_string(),+ repo: "app".to_string(),+ title: Some("Add pull request storage".to_string()),+ status: Some(GithubPullRequestStatus::Open),+ draft: true,+ author_github_user_id: None,+ author_login: None,+ requested_reviewer_github_user_ids: vec!["8".to_string()],+ participant_github_user_ids: vec!["42".to_string()],+ github_updated_at: None,+ }]+ );+}++#[tokio::test]+async fn upsert_after_a_repository_rename_moves_every_record_and_the_row() {+ let old_metadata = serde_json::to_value(EnrichedGithubPullRequest {+ repository_id: Some(99),+ ..pull_request(GithubPullRequestStatus::Open)+ })+ .unwrap();+ let team_record = stored_record(&team(), old_metadata.clone());+ let user_record = stored_record(&user(), old_metadata.clone());+ let foreign_entities =+ StubForeignEntityService::with_records(vec![team_record.clone(), user_record.clone()]);+ let rows = StubPullRequestRows::default();+ rows.rows+ .lock()+ .unwrap()+ .push(GithubPullRequestRow::from_metadata(&old_metadata).unwrap());+ let service = service(&foreign_entities, &rows);++ let upserted = service+ .upsert_pull_request(UpsertGithubPullRequest {+ pull_request: EnrichedGithubPullRequest {+ github_key: "macro/renamed/pull/7".to_string(),+ repo: "renamed".to_string(),+ url: "https://github.com/macro/renamed/pull/7".to_string(),+ display_name: "macro/renamed#7".to_string(),+ repository_id: Some(99),+ ..pull_request(GithubPullRequestStatus::Merged)+ },+ stored_for: team(),+ })+ .await+ .unwrap();++ assert!(foreign_entities.creates().is_empty());+ assert_eq!(upserted.foreign_entity.id, team_record.id);+ assert_eq!(+ upserted.previous_status,+ Some(GithubPullRequestStatus::Open)+ );+ assert!(+ foreign_entities+ .records()+ .iter()+ .all(|record| record.foreign_entity_id == "macro/renamed/pull/7")+ );+ let stored_rows = rows.rows();+ assert_eq!(stored_rows.len(), 1);+ assert_eq!(stored_rows[0].github_key, "macro/renamed/pull/7");+ assert_eq!(stored_rows[0].repo, "renamed");+ assert_eq!(stored_rows[0].status, Some(GithubPullRequestStatus::Merged));+}++#[tokio::test]+async fn index_writes_a_row_for_each_stored_pull_request_of_each_known_repository() {+ let metadata = |key: &str, number: u64, status: &str| {+ serde_json::json!({+ "githubKey": key,+ "owner": "Macro",+ "repo": "App",+ "number": number,+ "url": format!("https://github.com/{key}"),+ "displayName": key,+ "status": status,+ })+ };+ let rows = StubPullRequestRows::default();+ rows.stored.lock().unwrap().extend([+ (+ "macro".to_string(),+ "app".to_string(),+ metadata("Macro/App/pull/7", 7, "closed"),+ ),+ (+ "macro".to_string(),+ "app-web".to_string(),+ metadata("macro/app-web/pull/1", 1, "open"),+ ),+ ]);+ let service = service(&StubForeignEntityService::default(), &rows);++ let indexed = service+ .index_repositories(&[+ GithubRepositoryIdentity {+ id: 99,+ owner: "macro".to_string(),+ name: "app".to_string(),+ },+ GithubRepositoryIdentity {+ id: 0,+ owner: "macro".to_string(),+ name: "app-web".to_string(),+ },+ ])+ .await+ .unwrap();++ assert_eq!(indexed, 1);+ let stored_rows = rows.rows();+ assert_eq!(stored_rows.len(), 1);+ assert_eq!(stored_rows[0].github_key, "Macro/App/pull/7");+ assert_eq!(stored_rows[0].repository_id, Some(99));+ assert_eq!(stored_rows[0].status, Some(GithubPullRequestStatus::Closed));}diff --git a/crates/github_pull_requests/src/lib.rs b/crates/github_pull_requests/src/lib.rsindex fee5934598..ceb938ee0e 100644--- a/crates/github_pull_requests/src/lib.rs+++ b/crates/github_pull_requests/src/lib.rs@@ -1,12 +1,17 @@//! GitHub pull requests stored as foreign entity records.//!//! Owns the pull request model, the rules for merging refreshed pull request data into what is-//! already stored, and the service that writes pull request records.+//! already stored, the typed `github_pull_request` columns read from that data, and the service+//! that writes pull request records.//!//! # Architecture//!//! - **domain**: Contains domain models, ports, and service implementation.+//! - **outbound**: Contains adapters for external persistence systems.#![deny(missing_docs)]pub mod domain;++#[cfg(feature = "outbound")]+pub mod outbound;diff --git a/crates/github_pull_requests/src/outbound.rs b/crates/github_pull_requests/src/outbound.rsnew file mode 100644index 0000000000..9cdfba4f20--- /dev/null+++ b/crates/github_pull_requests/src/outbound.rs@@ -0,0 +1,3 @@+//! Outbound adapters for GitHub pull request storage.++pub mod pg_github_pull_request_repo;diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rsnew file mode 100644index 0000000000..5651ca1bd3--- /dev/null+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs@@ -0,0 +1,124 @@+//! PostgreSQL storage for the typed columns of GitHub pull requests.++mod index;+#[cfg(test)]+mod test;++use sqlx::PgPool;++use crate::domain::{models::GithubPullRequestRow, ports::GithubPullRequestRepository};++/// Stores pull request rows in the `github_pull_request` table.+#[derive(Clone)]+pub struct PgGithubPullRequestRepo {+ pool: PgPool,+}++impl PgGithubPullRequestRepo {+ /// Create a repository over `pool`.+ pub fn new(pool: PgPool) -> Self {+ Self { pool }+ }+}++impl GithubPullRequestRepository for PgGithubPullRequestRepo {+ type Err = sqlx::Error;++ #[tracing::instrument(err, skip(self))]+ async fn github_key_for(+ &self,+ repository_id: i64,+ number: i64,+ ) -> Result<Option<String>, Self::Err> {+ sqlx::query_scalar!(+ r#"+ SELECT github_key+ FROM github_pull_request+ WHERE repository_id = $1 AND number = $2+ "#,+ repository_id,+ number,+ )+ .fetch_optional(&self.pool)+ .await+ }++ #[tracing::instrument(err, skip(self, row), fields(github_key = %row.github_key))]+ async fn upsert_row(&self, row: &GithubPullRequestRow) -> Result<(), Self::Err> {+ sqlx::query!(+ r#"+ INSERT INTO github_pull_request (+ github_key,+ repository_id,+ number,+ owner,+ repo,+ title,+ status,+ draft,+ author_github_user_id,+ author_login,+ requested_reviewer_github_user_ids,+ participant_github_user_ids,+ github_updated_at+ )+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)+ ON CONFLICT (github_key) DO UPDATE SET+ repository_id = COALESCE(EXCLUDED.repository_id, github_pull_request.repository_id),+ number = EXCLUDED.number,+ owner = EXCLUDED.owner,+ repo = EXCLUDED.repo,+ title = EXCLUDED.title,+ status = EXCLUDED.status,+ draft = EXCLUDED.draft,+ author_github_user_id = EXCLUDED.author_github_user_id,+ author_login = EXCLUDED.author_login,+ requested_reviewer_github_user_ids = EXCLUDED.requested_reviewer_github_user_ids,+ participant_github_user_ids = EXCLUDED.participant_github_user_ids,+ github_updated_at = EXCLUDED.github_updated_at,+ updated_at = NOW()+ "#,+ row.github_key,+ row.repository_id,+ row.number,+ row.owner,+ row.repo,+ row.title.as_deref(),+ row.status.map(|status| status.as_str()),+ row.draft,+ row.author_github_user_id.as_deref(),+ row.author_login.as_deref(),+ &row.requested_reviewer_github_user_ids,+ &row.participant_github_user_ids,+ row.github_updated_at,+ )+ .execute(&self.pool)+ .await?;++ Ok(())+ }++ #[tracing::instrument(err, skip(self))]+ async fn rename_row(&self, from: &str, to: &str) -> Result<(), Self::Err> {+ sqlx::query!(+ r#"+ WITH taken AS (+ SELECT 1 FROM github_pull_request WHERE github_key = $2+ ),+ removed AS (+ DELETE FROM github_pull_request+ WHERE github_key = $1 AND EXISTS (SELECT 1 FROM taken)+ )+ UPDATE github_pull_request+ SET github_key = $2, updated_at = NOW()+ WHERE github_key = $1 AND NOT EXISTS (SELECT 1 FROM taken)+ "#,+ from,+ to,+ )+ .execute(&self.pool)+ .await?;++ Ok(())+ }+}diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rsnew file mode 100644index 0000000000..ce6119bf2b--- /dev/null+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs@@ -0,0 +1,36 @@+//! The stored pull request records the indexer writes rows from.++#[cfg(test)]+mod test;++use super::PgGithubPullRequestRepo;+use crate::domain::{+ models::GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, ports::GithubPullRequestIndexRepository,+};++/// The records belong to the foreign_entity crate; the indexer only reads them.+impl GithubPullRequestIndexRepository for PgGithubPullRequestRepo {+ type Err = sqlx::Error;++ #[tracing::instrument(err, skip(self))]+ async fn latest_pull_request_metadata(+ &self,+ owner: &str,+ name: &str,+ ) -> Result<Vec<serde_json::Value>, Self::Err> {+ let key_prefix = format!("{owner}/{name}/pull/");+ sqlx::query_scalar!(+ r#"+ SELECT DISTINCT ON (fe.foreign_entity_id) fe.metadata AS "metadata!: serde_json::Value"+ FROM foreign_entity fe+ WHERE fe.foreign_entity_source = $1::text+ AND left(lower(fe.foreign_entity_id), length($2::text)) = lower($2::text)+ ORDER BY fe.foreign_entity_id, fe.updated_at DESC, fe.id DESC+ "#,+ GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,+ key_prefix,+ )+ .fetch_all(&self.pool)+ .await+ }+}diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rsnew file mode 100644index 0000000000..4d78918985--- /dev/null+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs@@ -0,0 +1,129 @@+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use serde_json::{Value, json};+use sqlx::PgPool;++use super::super::PgGithubPullRequestRepo;+use crate::domain::ports::GithubPullRequestIndexRepository;++async fn insert_record(+ pool: &PgPool,+ key: &str,+ source: &str,+ stored_for_id: &str,+ metadata: Value,+ minutes_ago: i32,+) {+ sqlx::query(+ r#"+ INSERT INTO foreign_entity (+ id, foreign_entity_id, foreign_entity_source, metadata, stored_for_id,+ stored_for_auth_entity, updated_at+ )+ VALUES (gen_random_uuid(), $1, $2, $3, $4, 'user', NOW() - make_interval(mins => $5::int))+ "#,+ )+ .bind(key)+ .bind(source)+ .bind(metadata)+ .bind(stored_for_id)+ .bind(minutes_ago)+ .execute(pool)+ .await+ .expect("record should be inserted");+}++fn sorted(mut metadata: Vec<Value>) -> Vec<Value> {+ metadata.sort_by_key(|value| value.to_string());+ metadata+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn latest_metadata_covers_each_pull_request_in_the_repository_once(pool: PgPool) {+ let user = "macro|[email protected]";+ let team = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";+ insert_record(+ &pool,+ "Macro-Inc/App/pull/1",+ "github_pull_request",+ user,+ json!({ "n": 1 }),+ 0,+ )+ .await;+ insert_record(+ &pool,+ "macro-inc/app/pull/2",+ "github_pull_request",+ user,+ json!({ "n": "older" }),+ 10,+ )+ .await;+ insert_record(+ &pool,+ "macro-inc/app/pull/2",+ "github_pull_request",+ team,+ json!({ "n": "newer" }),+ 1,+ )+ .await;+ insert_record(+ &pool,+ "macro-inc/app-web/pull/3",+ "github_pull_request",+ user,+ json!({ "n": 3 }),+ 0,+ )+ .await;+ insert_record(+ &pool,+ "macro-inc/app/pull/4",+ "linear_issue",+ user,+ json!({ "n": 4 }),+ 0,+ )+ .await;++ let metadata = PgGithubPullRequestRepo::new(pool)+ .latest_pull_request_metadata("macro-inc", "app")+ .await+ .expect("records should load");++ assert_eq!(+ sorted(metadata),+ sorted(vec![json!({ "n": 1 }), json!({ "n": "newer" })])+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn repository_names_match_literally(pool: PgPool) {+ let user = "macro|[email protected]";+ insert_record(+ &pool,+ "macro-inc/appXrepo/pull/1",+ "github_pull_request",+ user,+ json!({ "n": 1 }),+ 0,+ )+ .await;+ insert_record(+ &pool,+ "macro-inc/app_repo/pull/2",+ "github_pull_request",+ user,+ json!({ "n": 2 }),+ 0,+ )+ .await;++ let metadata = PgGithubPullRequestRepo::new(pool)+ .latest_pull_request_metadata("macro-inc", "app_repo")+ .await+ .expect("records should load");++ assert_eq!(metadata, vec![json!({ "n": 2 })]);+}diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/test.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/test.rsnew file mode 100644index 0000000000..e53a9e57b1--- /dev/null+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/test.rs@@ -0,0 +1,122 @@+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;++use super::PgGithubPullRequestRepo;+use crate::domain::{+ models::{GithubPullRequestRow, GithubPullRequestStatus},+ ports::GithubPullRequestRepository,+};++async fn stored_row(+ pool: &PgPool,+ github_key: &str,+) -> Option<(Option<i64>, String, Option<String>)> {+ sqlx::query_as(+ r#"+ SELECT repository_id, repo, status+ FROM github_pull_request+ WHERE github_key = $1+ "#,+ )+ .bind(github_key)+ .fetch_optional(pool)+ .await+ .expect("row lookup should succeed")+}++fn row(github_key: &str, repository_id: Option<i64>) -> GithubPullRequestRow {+ GithubPullRequestRow {+ github_key: github_key.to_string(),+ repository_id,+ number: 7,+ owner: "macro".to_string(),+ repo: "app".to_string(),+ title: Some("Add pull request storage".to_string()),+ status: Some(GithubPullRequestStatus::Open),+ draft: false,+ author_github_user_id: Some("42".to_string()),+ author_login: Some("octocat".to_string()),+ requested_reviewer_github_user_ids: vec!["8".to_string()],+ participant_github_user_ids: vec!["8".to_string(), "42".to_string()],+ github_updated_at: None,+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn upsert_stores_a_row_found_by_repository_and_number(pool: PgPool) {+ let repo = PgGithubPullRequestRepo::new(pool.clone());++ repo.upsert_row(&row("macro/app/pull/7", Some(99)))+ .await+ .expect("upsert should succeed");++ assert_eq!(+ repo.github_key_for(99, 7).await.unwrap(),+ Some("macro/app/pull/7".to_string())+ );+ assert_eq!(repo.github_key_for(99, 8).await.unwrap(), None);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn upsert_without_a_repository_id_keeps_the_one_the_row_has(pool: PgPool) {+ let repo = PgGithubPullRequestRepo::new(pool.clone());++ repo.upsert_row(&row("macro/app/pull/7", None))+ .await+ .expect("upsert should succeed");+ assert_eq!(+ stored_row(&pool, "macro/app/pull/7").await,+ Some((None, "app".to_string(), Some("open".to_string())))+ );++ repo.upsert_row(&row("macro/app/pull/7", Some(99)))+ .await+ .expect("upsert should succeed");+ repo.upsert_row(&GithubPullRequestRow {+ status: Some(GithubPullRequestStatus::Merged),+ ..row("macro/app/pull/7", None)+ })+ .await+ .expect("upsert should succeed");++ assert_eq!(+ stored_row(&pool, "macro/app/pull/7").await,+ Some((Some(99), "app".to_string(), Some("merged".to_string())))+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn rename_moves_the_row_or_drops_it_when_the_new_key_has_one(pool: PgPool) {+ let repo = PgGithubPullRequestRepo::new(pool.clone());+ repo.upsert_row(&row("macro/app/pull/7", Some(99)))+ .await+ .unwrap();++ repo.rename_row("macro/app/pull/7", "macro/renamed/pull/7")+ .await+ .expect("rename should succeed");+ assert_eq!(stored_row(&pool, "macro/app/pull/7").await, None);+ assert_eq!(+ repo.github_key_for(99, 7).await.unwrap(),+ Some("macro/renamed/pull/7".to_string())+ );++ repo.upsert_row(&GithubPullRequestRow {+ number: 8,+ ..row("macro/app/pull/8", Some(99))+ })+ .await+ .unwrap();+ repo.upsert_row(&GithubPullRequestRow {+ number: 9,+ ..row("macro/renamed/pull/8", None)+ })+ .await+ .unwrap();+ repo.rename_row("macro/app/pull/8", "macro/renamed/pull/8")+ .await+ .expect("rename onto an existing row should succeed");++ assert_eq!(stored_row(&pool, "macro/app/pull/8").await, None);+ assert!(stored_row(&pool, "macro/renamed/pull/8").await.is_some());+}diff --git a/crates/macro_db_client/migrations/20260926173005_github_pull_request.sql b/crates/macro_db_client/migrations/20260926173005_github_pull_request.sqlnew file mode 100644index 0000000000..d3cada85c0--- /dev/null+++ b/crates/macro_db_client/migrations/20260926173005_github_pull_request.sql@@ -0,0 +1,32 @@+-- Typed columns for GitHub pull requests, so pull requests can be filtered and sorted without+-- reading JSON metadata. One row per pull request, shared by every foreign entity record stored+-- for it: a record joins its row on its owner/repo/pull/number key. Written by the+-- github_pull_requests crate from the records' metadata.+--+-- The repository's numeric id and the pull request number identify a pull request across+-- repository renames and transfers. The id is filled in once it is known.+CREATE TABLE github_pull_request (+ github_key TEXT PRIMARY KEY,+ repository_id BIGINT,+ number BIGINT NOT NULL,+ owner TEXT NOT NULL,+ repo TEXT NOT NULL,+ title TEXT,+ status TEXT CHECK (status IN ('open', 'closed', 'merged')),+ draft BOOLEAN NOT NULL DEFAULT FALSE,+ author_github_user_id TEXT,+ author_login TEXT,+ requested_reviewer_github_user_ids TEXT[] NOT NULL DEFAULT '{}',+ participant_github_user_ids TEXT[] NOT NULL DEFAULT '{}',+ github_updated_at TIMESTAMPTZ,+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),+ CONSTRAINT uq_github_pull_request_repository_number UNIQUE (repository_id, number)+);++CREATE INDEX idx_github_pull_request_author ON github_pull_request (author_github_user_id);+CREATE INDEX idx_github_pull_request_github_updated_at ON github_pull_request (github_updated_at DESC);+CREATE INDEX idx_github_pull_request_requested_reviewers+ ON github_pull_request USING GIN (requested_reviewer_github_user_ids);+CREATE INDEX idx_github_pull_request_participants+ ON github_pull_request USING GIN (participant_github_user_ids);diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.tomlindex 26c69950b7..de7d6a180c 100644--- a/services/authentication_service/Cargo.toml+++ b/services/authentication_service/Cargo.toml@@ -85,6 +85,7 @@ github = { path = "../../crates/github", default-features = false, features = ["outbound",] }github_pull_requests = { path = "../../crates/github_pull_requests", default-features = false, features = [+ "outbound","service",] }http-body-util = { workspace = true }diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rsindex 0d832129c1..030330c184 100644--- a/services/authentication_service/src/api/context.rs+++ b/services/authentication_service/src/api/context.rs@@ -24,7 +24,10 @@ use github::domain::service::GithubLinkServiceImpl;use github::outbound::github_auth_client::GithubAuthImpl;use github::outbound::github_oauth_client::GithubOauthImpl;use github::outbound::pg_github_repo::PgGithubRepo;-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;+use github_pull_requests::{+ domain::service::GithubPullRequestServiceImpl,+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,+};use gtm_invite::{domain::service::GtmInviteServiceImpl, outbound::pg_gtm_invite_repo::PgGtmInviteRepo,};@@ -118,7 +121,10 @@ pub(crate) type GithubLinkServiceType = GithubLinkServiceImpl<PgGithubRepo,GithubOauthImpl,GithubAuthImpl,- GithubPullRequestServiceImpl<ForeignEntityServiceImpl<PgForeignEntityRepo>>,+ GithubPullRequestServiceImpl<+ ForeignEntityServiceImpl<PgForeignEntityRepo>,+ PgGithubPullRequestRepo,+ >,>;pub(crate) type EntityAccessServiceType = EntityAccessServiceImpl<PgAccessRepository>;diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rsindex adec62a2f3..c9c7458b04 100644--- a/services/authentication_service/src/main.rs+++ b/services/authentication_service/src/main.rs@@ -32,7 +32,10 @@ use github::{pg_github_repo::PgGithubRepo,},};-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;+use github_pull_requests::{+ domain::service::GithubPullRequestServiceImpl,+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,+};use loops_client::LoopsClient;use macro_auth::middleware::decode_jwt::JwtValidationArgs;use macro_authorization::{@@ -454,7 +457,10 @@ async fn main() -> anyhow::Result<()> {PgGithubRepo::new(db.clone()),GithubOauthImpl::default(),GithubAuthImpl::new(auth_client.clone(), redis_multiplexed_conn),- GithubPullRequestServiceImpl::new(foreign_entity_service),+ GithubPullRequestServiceImpl::new(+ foreign_entity_service,+ PgGithubPullRequestRepo::new(db.clone()),+ ),GithubLinkConfig {client_id: config.github_client_id.to_string(),client_secret: config.github_client_secret.to_string(),diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex ab41e9e51a..c3a3028561 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -131,6 +131,7 @@ github = { path = "../../crates/github", default-features = false, features = ["sync",] }github_pull_requests = { path = "../../crates/github_pull_requests", default-features = false, features = [+ "outbound","service",] }graphql_soup = { path = "../../crates/graphql_soup" }diff --git a/services/document_storage_service/src/api.rs b/services/document_storage_service/src/api.rsindex 7cbbc6c041..b2b1b2ced0 100644--- a/services/document_storage_service/src/api.rs+++ b/services/document_storage_service/src/api.rs@@ -308,6 +308,12 @@ fn api_router(state: ApiContext) -> Router {"/internal",internal::router(state.clone()).nest("/notifications", notification::router())+ .nest(+ "/github",+ github::inbound::pull_request_index_router::pull_request_index_router(+ state.github_pull_request_index_state.clone(),+ ),+ ).nest("/search",search_service::search_router()diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex ff03cdf01d..9ac9dcb700 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -87,11 +87,15 @@ use foreign_entity::{outbound::pg_foreign_entity_repo::PgForeignEntityRepo,};use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};-use github::domain::service::GithubSyncServiceImpl;+use github::domain::service::{GithubSyncServiceImpl, PullRequestIndexService};+use github::inbound::pull_request_index_router::PullRequestIndexRouterState;use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;use github::outbound::github_sync_client::GithubSyncClientImpl;use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;+use github_pull_requests::{+ domain::service::GithubPullRequestServiceImpl,+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,+};use initiative::{domain::service::InitiativeServiceImpl, inbound::axum_router::InitiativeRouterState,outbound::PgInitiativeRepo,@@ -659,16 +663,28 @@ pub(crate) type ForeignEntityServiceType = ForeignEntityServiceImpl<PgForeignEntpub(crate) type DssForeignEntityState =ForeignEntityRouterState<ForeignEntityServiceType, EntityAccessService, AuthorizationService>;+/// Type alias for the GitHub pull request service.+pub(crate) type GithubPullRequestServiceType =+ GithubPullRequestServiceImpl<ForeignEntityServiceType, PgGithubPullRequestRepo>;+/// Type alias for the github sync service.pub(crate) type GithubSyncServiceType = GithubSyncServiceImpl<DocumentService,PgGithubSyncRepo,GithubSyncClientImpl,- GithubPullRequestServiceImpl<ForeignEntityServiceType>,+ GithubPullRequestServiceType,NotificationIngressType,ConnectionGatewayGithubRealtime,>;+/// Type alias for the GitHub pull request index service.+pub(crate) type GithubPullRequestIndexServiceType =+ PullRequestIndexService<PgGithubSyncRepo, GithubSyncClientImpl, GithubPullRequestServiceType>;++/// Type alias for the GitHub pull request index router state.+pub(crate) type DssGithubPullRequestIndexState =+ PullRequestIndexRouterState<GithubPullRequestIndexServiceType, AuthorizationService>;+/// Type alias for the cal.com webhook service.pub(crate) type CalWebhookServiceType = CalWebhookServiceImpl<AnalyticsClientSink>;@@ -713,6 +729,7 @@ pub(crate) struct ApiContext {pub redis_client: Arc<Redis>,pub s3_client: Arc<S3>,pub github_sync_service: Arc<GithubSyncServiceType>,+ pub github_pull_request_index_state: DssGithubPullRequestIndexState,pub dynamodb_client: Arc<DynamodbClient>,pub dynamo_db: aws_sdk_dynamodb::Client,pub soup_router_state: DssSoupState,diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 041b18332b..06cfde2111 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -89,11 +89,17 @@ use foreign_entity::{outbound::pg_foreign_entity_repo::PgForeignEntityRepo,};use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};-use github::domain::service::{GithubSyncConfig, GithubSyncServiceImpl};+use github::domain::service::{+ GithubSyncConfig, GithubSyncServiceImpl, InstallationTokenConfig, PullRequestIndexService,+};+use github::inbound::pull_request_index_router::PullRequestIndexRouterState;use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;use github::outbound::github_sync_client::GithubSyncClientImpl;use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;+use github_pull_requests::{+ domain::service::GithubPullRequestServiceImpl,+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,+};use graphql_scheduled_action::ScheduledActionGraphqlContext;use harnesses::outbound::pg_harness_repo::PgHarnessRepo;use initiative::{@@ -577,6 +583,7 @@ async fn run() -> anyhow::Result<()> {document_service.clone(),Arc::new(GithubPullRequestServiceImpl::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),)),(*notification_ingress_service).clone(),PgGithubSyncRepo::new(db.clone()),@@ -584,6 +591,22 @@ async fn run() -> anyhow::Result<()> {ConnectionGatewayGithubRealtime::new(conn_gateway_client.clone()),);+ let github_pull_request_index_state = PullRequestIndexRouterState {+ service: Arc::new(PullRequestIndexService::new(+ InstallationTokenConfig {+ client_id: config.github_sync_app_client_id.to_string(),+ private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),+ },+ PgGithubSyncRepo::new(db.clone()),+ GithubSyncClientImpl::default(),+ GithubPullRequestServiceImpl::new(+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ PgGithubPullRequestRepo::new(db.clone()),+ ),+ )),+ authorization_state: authorization_state.clone(),+ };+let foreign_entity_state = ForeignEntityRouterState::new(foreign_entity_service.clone(),entity_access_service.clone(),@@ -1816,6 +1839,7 @@ async fn run() -> anyhow::Result<()> {)),graphql_entity_mutation_service,github_sync_service: Arc::new(github_sync_service_impl),+ github_pull_request_index_state,foreign_entity_state,db: db.clone(),readonly_db: readonly_pool::ReadOnlyPool(readonly_db.clone()),