Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(github_pull_requests): store typed pull request columns in github_pull_request (#7103)

dev-rbf0f5b4f5f7f6
diff --git a/.sqlx/query-1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf.json b/.sqlx/query-1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf.json
new file mode 100644
index 0000000000..f960121345
--- /dev/null
+++ b/.sqlx/query-1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH taken AS (\n SELECT 1 FROM github_pull_request WHERE github_key = $2\n ),\n removed AS (\n DELETE FROM github_pull_request\n WHERE github_key = $1 AND EXISTS (SELECT 1 FROM taken)\n )\n UPDATE github_pull_request\n SET github_key = $2, updated_at = NOW()\n WHERE github_key = $1 AND NOT EXISTS (SELECT 1 FROM taken)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "1ed57b03ee6be255aa5088514879e783020c679ffd30c72ccdfcbb6e1c5a04bf"
+}
diff --git a/.sqlx/query-45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9.json b/.sqlx/query-45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9.json
new file mode 100644
index 0000000000..a7e8cea936
--- /dev/null
+++ b/.sqlx/query-45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9.json
@@ -0,0 +1,26 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO github_pull_request (\n github_key,\n repository_id,\n number,\n owner,\n repo,\n title,\n status,\n draft,\n author_github_user_id,\n author_login,\n requested_reviewer_github_user_ids,\n participant_github_user_ids,\n github_updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)\n ON CONFLICT (github_key) DO UPDATE SET\n repository_id = COALESCE(EXCLUDED.repository_id, github_pull_request.repository_id),\n number = EXCLUDED.number,\n owner = EXCLUDED.owner,\n repo = EXCLUDED.repo,\n title = EXCLUDED.title,\n status = EXCLUDED.status,\n draft = EXCLUDED.draft,\n author_github_user_id = EXCLUDED.author_github_user_id,\n author_login = EXCLUDED.author_login,\n requested_reviewer_github_user_ids = EXCLUDED.requested_reviewer_github_user_ids,\n participant_github_user_ids = EXCLUDED.participant_github_user_ids,\n github_updated_at = EXCLUDED.github_updated_at,\n updated_at = NOW()\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8",
+ "Int8",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Bool",
+ "Text",
+ "Text",
+ "TextArray",
+ "TextArray",
+ "Timestamptz"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "45c058694137c5390aa599934b7d8fc3dcb5812605c1aa15a2787135be4997d9"
+}
diff --git a/.sqlx/query-58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6.json b/.sqlx/query-58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6.json
new file mode 100644
index 0000000000..fa1eabf5f9
--- /dev/null
+++ b/.sqlx/query-58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT id AS \"id!\"\n FROM github_app_installation\n WHERE $1::text IS NULL OR id > $1\n ORDER BY id\n LIMIT $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id!",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "58f4809821bd958ca59df997bca0f3c4e606e7735c219c6884bb4e20a6cdd8c6"
+}
diff --git a/.sqlx/query-a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65.json b/.sqlx/query-a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65.json
new file mode 100644
index 0000000000..fc15767c03
--- /dev/null
+++ b/.sqlx/query-a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT github_key\n FROM github_pull_request\n WHERE repository_id = $1 AND number = $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "github_key",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "a99d4d5ff6c6f2dfc87cb6b14b4b6dc0b57f7f540408aa20d48965e510244c65"
+}
diff --git a/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json b/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json
new file mode 100644
index 0000000000..e8fb0d829f
--- /dev/null
+++ b/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT ON (fe.foreign_entity_id) fe.metadata AS \"metadata!: serde_json::Value\"\n FROM foreign_entity fe\n WHERE fe.foreign_entity_source = $1::text\n AND left(lower(fe.foreign_entity_id), length($2::text)) = lower($2::text)\n ORDER BY fe.foreign_entity_id, fe.updated_at DESC, fe.id DESC\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "metadata!: serde_json::Value",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22"
+}
diff --git a/Cargo.lock b/Cargo.lock
index 27092a8b99..d53eeacc4d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8142,11 +8142,14 @@ dependencies = [
name = "github_pull_requests"
version = "0.1.0"
dependencies = [
+ "anyhow",
"chrono",
"entity_access",
"foreign_entity",
+ "macro_db_migrator",
"serde",
"serde_json",
+ "sqlx",
"thiserror 2.0.18",
"tokio",
"tracing",
diff --git a/crates/github/src/domain/models.rs b/crates/github/src/domain/models.rs
index 109e2ea759..1a97dae2bd 100644
--- a/crates/github/src/domain/models.rs
+++ b/crates/github/src/domain/models.rs
@@ -24,7 +24,7 @@ pub use installation_state::{
InstallationState, InstallationStateError, sign_installation_state, verify_installation_state,
};
pub use link::{GithubAccessToken, GithubExchangeTokenResponse, GithubLink, GithubUserInfo};
-pub use repository::GithubRepository;
+pub use repository::{GithubRepository, PullRequestIndexPage, PullRequestIndexRequest};
pub use sync::{
GithubAppInstallationSource, GithubAuthenticatedUser, GithubInstallationAccessToken,
GithubInstallationSetupAction, GithubSetupAccessToken, GithubUserInstallation,
diff --git a/crates/github/src/domain/models/repository.rs b/crates/github/src/domain/models/repository.rs
index 30a54d1d59..e5836765d3 100644
--- a/crates/github/src/domain/models/repository.rs
+++ b/crates/github/src/domain/models/repository.rs
@@ -18,6 +18,10 @@ pub struct GithubRepository {
pub default_branch: Option<String>,
/// Whether the repository is private.
pub private: bool,
+ /// GitHub's numeric repository id, which survives renames and transfers. Zero when it was
+ /// not recorded.
+ #[serde(default)]
+ pub id: u64,
}
impl GithubRepository {
@@ -29,3 +33,31 @@ impl GithubRepository {
format!("https://github.com/{}/{}", self.owner, self.name)
}
}
+
+/// A request for the next page of installations whose stored pull requests to index.
+#[derive(Debug, Clone, Default, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PullRequestIndexRequest {
+ /// Resume after this installation id; absent to start from the first installation.
+ #[serde(default)]
+ pub after: Option<String>,
+ /// How many installations to process in this call; absent for the default page size.
+ #[serde(default)]
+ pub limit: Option<u32>,
+}
+
+/// What one page of pull request indexing did.
+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PullRequestIndexPage {
+ /// Installations processed in this page, including those that failed.
+ pub installations: u32,
+ /// Repositories listed across the page's installations.
+ pub repositories: u32,
+ /// Pull request rows written.
+ pub indexed_pull_requests: u64,
+ /// Installations whose repositories GitHub would not list, such as suspended ones.
+ pub failed_installation_ids: Vec<String>,
+ /// Pass as `after` to continue; absent once every installation has been processed.
+ pub next_after: Option<String>,
+}
diff --git a/crates/github/src/domain/ports/mod.rs b/crates/github/src/domain/ports/mod.rs
index c8a9ff1a1c..4746fd5347 100644
--- a/crates/github/src/domain/ports/mod.rs
+++ b/crates/github/src/domain/ports/mod.rs
@@ -11,5 +11,6 @@ mod sync;
pub use link::{Auth, GithubLinkService, GithubOauth, GithubRepo};
#[cfg(feature = "sync")]
pub use sync::{
- GithubRepositoryClient, GithubSyncClient, GithubSyncRealtime, GithubSyncRepo, GithubSyncService,
+ GithubInstallationLister, GithubPullRequestIndex, GithubRepositoryClient, GithubSyncClient,
+ GithubSyncRealtime, GithubSyncRepo, GithubSyncService,
};
diff --git a/crates/github/src/domain/ports/sync.rs b/crates/github/src/domain/ports/sync.rs
index 6fa456a3c3..a68b0c22d1 100644
--- a/crates/github/src/domain/ports/sync.rs
+++ b/crates/github/src/domain/ports/sync.rs
@@ -8,7 +8,8 @@ use crate::domain::models::{
AppJwt, EnrichedGithubPullRequest, GithubAppInstallationSource, GithubAuthenticatedUser,
GithubError, GithubInstallationAccessToken, GithubKey, GithubPullRequestDetails,
GithubRepository, GithubSetupAccessToken, GithubUserInstallation, MacroTaskId,
- ResolvedTeamTaskReference, TeamTaskReference, ValidatedGithubWebhookEvent,
+ PullRequestIndexPage, PullRequestIndexRequest, ResolvedTeamTaskReference, TeamTaskReference,
+ ValidatedGithubWebhookEvent,
};
/// Repository for accessing github sync data from the database.
@@ -310,6 +311,29 @@ pub trait GithubSyncRealtime: Send + Sync + 'static {
) -> impl Future<Output = Result<(), GithubError>> + Send;
}
+/// Pages through every GitHub App installation Macro has recorded.
+pub trait GithubInstallationLister: Send + Sync + 'static {
+ /// The error type returned by repository operations.
+ type Err: Into<anyhow::Error> + Send + std::fmt::Debug;
+
+ /// Up to `limit` distinct installation ids that sort after `after`, in ascending order.
+ fn list_installation_ids(
+ &self,
+ after: Option<&str>,
+ limit: u32,
+ ) -> impl Future<Output = Result<Vec<String>, Self::Err>> + Send;
+}
+
+/// Writes typed rows for pull requests stored before rows existed, one page of installations at
+/// a time.
+pub trait GithubPullRequestIndex: Send + Sync + 'static {
+ /// Process the next page of installations and report what changed.
+ fn index_pull_requests(
+ &self,
+ request: PullRequestIndexRequest,
+ ) -> impl Future<Output = Result<PullRequestIndexPage, GithubError>> + Send;
+}
+
/// Lists repository metadata for an installation the caller is authorized to access.
pub trait GithubRepositoryClient: Send + Sync + 'static {
/// Fetch the installation's repositories using metadata-only access.
diff --git a/crates/github/src/domain/service/link_test.rs b/crates/github/src/domain/service/link_test.rs
index e35d20e0d5..55d6d86f43 100644
--- a/crates/github/src/domain/service/link_test.rs
+++ b/crates/github/src/domain/service/link_test.rs
@@ -8,7 +8,10 @@ use foreign_entity::domain::{
},
ports::{ForeignEntityListQuery, ForeignEntityService},
};
-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;
+use github_pull_requests::domain::{
+ models::GithubPullRequestRow, ports::GithubPullRequestRepository,
+ service::GithubPullRequestServiceImpl,
+};
use macro_user_id::{
lowercased::Lowercase,
user_id::{MacroUserId, MacroUserIdStr},
@@ -210,6 +213,7 @@ fn default_pull_request_details() -> GithubPullRequestDetails {
GithubPullRequestDetails {
title: "Add token validation".to_string(),
state: "open".to_string(),
+ repository_id: None,
merged_at: None,
additions: 12,
deletions: 3,
@@ -219,6 +223,9 @@ fn default_pull_request_details() -> GithubPullRequestDetails {
comments: None,
checks: None,
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}
}
@@ -597,9 +604,31 @@ type TestGithubLinkService = GithubLinkServiceImpl<
StubGithubRepo,
StubGithubOauth,
StubAuth,
- GithubPullRequestServiceImpl<StubForeignEntityService>,
+ GithubPullRequestServiceImpl<StubForeignEntityService, NoPullRequestRows>,
>;
+struct NoPullRequestRows;
+
+impl GithubPullRequestRepository for NoPullRequestRows {
+ type Err = std::convert::Infallible;
+
+ async fn github_key_for(
+ &self,
+ _repository_id: i64,
+ _number: i64,
+ ) -> Result<Option<String>, Self::Err> {
+ Ok(None)
+ }
+
+ async fn upsert_row(&self, _row: &GithubPullRequestRow) -> Result<(), Self::Err> {
+ Ok(())
+ }
+
+ async fn rename_row(&self, _from: &str, _to: &str) -> Result<(), Self::Err> {
+ Ok(())
+ }
+}
+
fn service(repo: StubGithubRepo, oauth: StubGithubOauth, auth: StubAuth) -> TestGithubLinkService {
service_with_foreign_entities(repo, oauth, auth, StubForeignEntityService::default())
}
@@ -614,7 +643,7 @@ fn service_with_foreign_entities(
repo,
oauth,
auth,
- GithubPullRequestServiceImpl::new(foreign_entity_service),
+ GithubPullRequestServiceImpl::new(foreign_entity_service, NoPullRequestRows),
GithubLinkConfig {
client_id: "client-id".to_string(),
client_secret: "client-secret".to_string(),
diff --git a/crates/github/src/domain/service/mod.rs b/crates/github/src/domain/service/mod.rs
index a563118439..f8bda7f691 100644
--- a/crates/github/src/domain/service/mod.rs
+++ b/crates/github/src/domain/service/mod.rs
@@ -3,6 +3,8 @@
#[cfg(feature = "sync")]
mod installation_tokens;
#[cfg(feature = "sync")]
+mod pull_request_index;
+#[cfg(feature = "sync")]
mod reachable_repositories;
#[cfg(feature = "sync")]
mod sync;
@@ -10,6 +12,8 @@ mod sync;
#[cfg(feature = "sync")]
pub use installation_tokens::{InstallationTokenConfig, InstallationTokenService};
#[cfg(feature = "sync")]
+pub use pull_request_index::PullRequestIndexService;
+#[cfg(feature = "sync")]
pub use reachable_repositories::ReachableRepositoriesService;
#[cfg(feature = "sync")]
pub use sync::{GithubSyncConfig, GithubSyncServiceImpl};
diff --git a/crates/github/src/domain/service/pull_request_index.rs b/crates/github/src/domain/service/pull_request_index.rs
new file mode 100644
index 0000000000..ae2f2ab476
--- /dev/null
+++ b/crates/github/src/domain/service/pull_request_index.rs
@@ -0,0 +1,160 @@
+//! Writing typed rows for pull requests stored before rows existed.
+//!
+//! A pull request row is identified by its repository's numeric id, which older records do not
+//! carry. This pages through every installation, asks GitHub which repositories it covers, and
+//! has the pull request store write a row for each record synced under one of those
+//! repositories' names. Rows are rewritten from the record each time, so a page can be re-run
+//! safely.
+
+use github_pull_requests::domain::{
+ models::GithubRepositoryIdentity, ports::GithubPullRequestIndexer,
+};
+
+use crate::domain::models::{
+ AppJwt, GithubError, PullRequestIndexPage, PullRequestIndexRequest, app_jwt,
+};
+use crate::domain::ports::{
+ GithubInstallationLister, GithubPullRequestIndex, GithubRepositoryClient,
+};
+
+use super::InstallationTokenConfig;
+
+#[cfg(test)]
+mod test;
+
+/// Installations processed per call when the request does not say.
+const DEFAULT_PAGE_SIZE: u32 = 25;
+/// The most installations one call processes, keeping a call well inside request timeouts.
+const MAX_PAGE_SIZE: u32 = 100;
+
+/// Indexes stored pull requests one page of installations at a time.
+pub struct PullRequestIndexService<Installations, Client, PullRequests> {
+ config: InstallationTokenConfig,
+ installations: Installations,
+ client: Client,
+ pull_requests: PullRequests,
+}
+
+impl<Installations, Client, PullRequests>
+ PullRequestIndexService<Installations, Client, PullRequests>
+{
+ /// Build the service over the App's credentials, the recorded installations, a GitHub
+ /// client, and the pull request store.
+ pub fn new(
+ config: InstallationTokenConfig,
+ installations: Installations,
+ client: Client,
+ pull_requests: PullRequests,
+ ) -> Self {
+ Self {
+ config,
+ installations,
+ client,
+ pull_requests,
+ }
+ }
+}
+
+impl<Installations, Client, PullRequests> GithubPullRequestIndex
+ for PullRequestIndexService<Installations, Client, PullRequests>
+where
+ Installations: GithubInstallationLister,
+ Client: GithubRepositoryClient,
+ PullRequests: GithubPullRequestIndexer,
+{
+ #[tracing::instrument(skip(self), err)]
+ async fn index_pull_requests(
+ &self,
+ request: PullRequestIndexRequest,
+ ) -> Result<PullRequestIndexPage, GithubError> {
+ let limit = request
+ .limit
+ .unwrap_or(DEFAULT_PAGE_SIZE)
+ .clamp(1, MAX_PAGE_SIZE);
+ let installation_ids = self
+ .installations
+ .list_installation_ids(request.after.as_deref(), limit)
+ .await
+ .map_err(|error| {
+ GithubError::Internal(anyhow::anyhow!("could not list installations: {error:?}"))
+ })?;
+
+ // A full page may have more after it; a short one is the last.
+ let next_after = if installation_ids.len() == limit as usize {
+ installation_ids.last().cloned()
+ } else {
+ None
+ };
+ let mut page = PullRequestIndexPage {
+ installations: 0,
+ repositories: 0,
+ indexed_pull_requests: 0,
+ failed_installation_ids: Vec::new(),
+ next_after,
+ };
+ if installation_ids.is_empty() {
+ return Ok(page);
+ }
+
+ let jwt = app_jwt(&self.config.client_id, &self.config.private_key_pem)?;
+ for installation_id in installation_ids {
+ page.installations += 1;
+ let Some(repositories) = self.list_repositories(&jwt, &installation_id).await else {
+ page.failed_installation_ids.push(installation_id);
+ continue;
+ };
+ page.repositories += u32::try_from(repositories.len()).unwrap_or(u32::MAX);
+ page.indexed_pull_requests += self
+ .pull_requests
+ .index_repositories(&repositories)
+ .await
+ .map_err(|error| {
+ GithubError::Internal(anyhow::anyhow!(
+ "could not index pull requests for installation {installation_id}: {error}"
+ ))
+ })?;
+ }
+
+ Ok(page)
+ }
+}
+
+impl<Installations, Client, PullRequests>
+ PullRequestIndexService<Installations, Client, PullRequests>
+where
+ Client: GithubRepositoryClient,
+{
+ /// The installation's repositories, or `None` when GitHub will not list them, for example
+ /// because the installation was suspended or removed.
+ async fn list_repositories(
+ &self,
+ jwt: &AppJwt,
+ installation_id: &str,
+ ) -> Option<Vec<GithubRepositoryIdentity>> {
+ let installation = installation_id
+ .parse::<u64>()
+ .inspect_err(|error| {
+ tracing::warn!(error=?error, installation_id, "installation id is not a number");
+ })
+ .ok()?;
+ let repositories = self
+ .client
+ .repositories_for_installation(jwt, installation)
+ .await
+ .inspect_err(|error| {
+ tracing::warn!(error=?error, installation_id, "could not list installation repositories");
+ })
+ .ok()?;
+
+ Some(
+ repositories
+ .into_iter()
+ .map(|repository| GithubRepositoryIdentity {
+ id: repository.id,
+ owner: repository.owner,
+ name: repository.name,
+ })
+ .collect(),
+ )
+ }
+}
diff --git a/crates/github/src/domain/service/pull_request_index/test.rs b/crates/github/src/domain/service/pull_request_index/test.rs
new file mode 100644
index 0000000000..ac19880ad7
--- /dev/null
+++ b/crates/github/src/domain/service/pull_request_index/test.rs
@@ -0,0 +1,258 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+
+use github_pull_requests::domain::models::GithubPullRequestError;
+
+use super::*;
+use crate::domain::models::GithubRepository;
+
+const TEST_PEM: &str = include_str!("../installation_tokens/test_key.pem");
+
+fn config() -> InstallationTokenConfig {
+ InstallationTokenConfig {
+ client_id: "Iv1.testclient".to_owned(),
+ private_key_pem: TEST_PEM.to_owned(),
+ }
+}
+
+fn repository(id: u64, owner: &str, name: &str) -> GithubRepository {
+ GithubRepository {
+ owner: owner.to_owned(),
+ name: name.to_owned(),
+ html_url: format!("https://github.com/{owner}/{name}"),
+ default_branch: Some("main".to_owned()),
+ private: true,
+ id,
+ }
+}
+
+fn identity(id: u64, owner: &str, name: &str) -> GithubRepositoryIdentity {
+ GithubRepositoryIdentity {
+ id,
+ owner: owner.to_owned(),
+ name: name.to_owned(),
+ }
+}
+
+struct FakeInstallations(Vec<String>);
+
+impl GithubInstallationLister for FakeInstallations {
+ type Err = anyhow::Error;
+
+ async fn list_installation_ids(
+ &self,
+ after: Option<&str>,
+ limit: u32,
+ ) -> Result<Vec<String>, Self::Err> {
+ Ok(self
+ .0
+ .iter()
+ .filter(|id| after.is_none_or(|after| id.as_str() > after))
+ .take(limit as usize)
+ .cloned()
+ .collect())
+ }
+}
+
+#[derive(Default)]
+struct FakeClient {
+ repositories: HashMap<u64, Vec<GithubRepository>>,
+ unavailable: Vec<u64>,
+}
+
+impl GithubRepositoryClient for FakeClient {
+ async fn repositories_for_installation(
+ &self,
+ _jwt: &AppJwt,
+ installation_id: u64,
+ ) -> Result<Vec<GithubRepository>, GithubError> {
+ if self.unavailable.contains(&installation_id) {
+ return Err(GithubError::Internal(anyhow::anyhow!(
+ "installation suspended"
+ )));
+ }
+ Ok(self
+ .repositories
+ .get(&installation_id)
+ .cloned()
+ .unwrap_or_default())
+ }
+}
+
+/// Records every batch it is given and reports one row per repository.
+#[derive(Clone, Default)]
+struct RecordingPullRequests {
+ batches: Arc<Mutex<Vec<Vec<GithubRepositoryIdentity>>>>,
+ fail: bool,
+}
+
+impl RecordingPullRequests {
+ fn batches(&self) -> Vec<Vec<GithubRepositoryIdentity>> {
+ self.batches.lock().expect("lock").clone()
+ }
+}
+
+impl GithubPullRequestIndexer for RecordingPullRequests {
+ async fn index_repositories(
+ &self,
+ repositories: &[GithubRepositoryIdentity],
+ ) -> Result<u64, GithubPullRequestError> {
+ if self.fail {
+ return Err(GithubPullRequestError::Repository(anyhow::anyhow!(
+ "database unavailable"
+ )));
+ }
+ self.batches
+ .lock()
+ .expect("lock")
+ .push(repositories.to_vec());
+ Ok(repositories.len() as u64)
+ }
+}
+
+fn service(
+ installations: &[&str],
+ client: FakeClient,
+ pull_requests: RecordingPullRequests,
+) -> PullRequestIndexService<FakeInstallations, FakeClient, RecordingPullRequests> {
+ PullRequestIndexService::new(
+ config(),
+ FakeInstallations(installations.iter().map(|id| id.to_string()).collect()),
+ client,
+ pull_requests,
+ )
+}
+
+fn two_installation_client() -> FakeClient {
+ FakeClient {
+ repositories: HashMap::from([
+ (
+ 1,
+ vec![
+ repository(100, "macro-inc", "macro"),
+ repository(200, "macro-inc", "docs"),
+ ],
+ ),
+ (3, vec![repository(300, "other-org", "tools")]),
+ ]),
+ unavailable: Vec::new(),
+ }
+}
+
+#[tokio::test]
+async fn pages_through_installations_and_reports_where_to_resume() {
+ let pull_requests = RecordingPullRequests::default();
+ let service = service(
+ &["1", "2", "3"],
+ two_installation_client(),
+ pull_requests.clone(),
+ );
+
+ let first = service
+ .index_pull_requests(PullRequestIndexRequest {
+ after: None,
+ limit: Some(2),
+ })
+ .await
+ .expect("first page should succeed");
+ assert_eq!(
+ first,
+ PullRequestIndexPage {
+ installations: 2,
+ repositories: 2,
+ indexed_pull_requests: 2,
+ failed_installation_ids: Vec::new(),
+ next_after: Some("2".to_string()),
+ }
+ );
+
+ let second = service
+ .index_pull_requests(PullRequestIndexRequest {
+ after: first.next_after,
+ limit: Some(2),
+ })
+ .await
+ .expect("second page should succeed");
+ assert_eq!(
+ second,
+ PullRequestIndexPage {
+ installations: 1,
+ repositories: 1,
+ indexed_pull_requests: 1,
+ failed_installation_ids: Vec::new(),
+ next_after: None,
+ }
+ );
+
+ assert_eq!(
+ pull_requests.batches(),
+ vec![
+ vec![
+ identity(100, "macro-inc", "macro"),
+ identity(200, "macro-inc", "docs"),
+ ],
+ vec![],
+ vec![identity(300, "other-org", "tools")],
+ ]
+ );
+}
+
+#[tokio::test]
+async fn installations_github_will_not_list_are_reported_and_skipped() {
+ let pull_requests = RecordingPullRequests::default();
+ let client = FakeClient {
+ unavailable: vec![1],
+ ..two_installation_client()
+ };
+ let service = service(&["1", "3"], client, pull_requests.clone());
+
+ let page = service
+ .index_pull_requests(PullRequestIndexRequest::default())
+ .await
+ .expect("page should succeed");
+
+ assert_eq!(page.failed_installation_ids, vec!["1".to_string()]);
+ assert_eq!(page.installations, 2);
+ assert_eq!(page.indexed_pull_requests, 1);
+ assert_eq!(page.next_after, None);
+ assert_eq!(
+ pull_requests.batches(),
+ vec![vec![identity(300, "other-org", "tools")]]
+ );
+}
+
+#[tokio::test]
+async fn a_storage_failure_stops_the_page() {
+ let pull_requests = RecordingPullRequests {
+ fail: true,
+ ..RecordingPullRequests::default()
+ };
+ let service = service(&["1"], two_installation_client(), pull_requests);
+
+ let error = service
+ .index_pull_requests(PullRequestIndexRequest::default())
+ .await
+ .expect_err("a storage failure should fail the page");
+
+ assert!(matches!(error, GithubError::Internal(_)));
+}
+
+#[tokio::test]
+async fn a_zero_limit_still_processes_one_installation() {
+ let service = service(
+ &["1", "3"],
+ two_installation_client(),
+ RecordingPullRequests::default(),
+ );
+
+ let page = service
+ .index_pull_requests(PullRequestIndexRequest {
+ after: None,
+ limit: Some(0),
+ })
+ .await
+ .expect("page should succeed");
+
+ assert_eq!(page.installations, 1);
+ assert_eq!(page.next_after, Some("1".to_string()));
+}
diff --git a/crates/github/src/domain/service/reachable_repositories/test.rs b/crates/github/src/domain/service/reachable_repositories/test.rs
index e241ed86c8..bd9cd04b42 100644
--- a/crates/github/src/domain/service/reachable_repositories/test.rs
+++ b/crates/github/src/domain/service/reachable_repositories/test.rs
@@ -30,6 +30,7 @@ fn repository(owner: &str, name: &str) -> GithubRepository {
html_url: format!("https://github.com/{owner}/{name}"),
default_branch: Some("main".to_owned()),
private: true,
+ id: 0,
}
}
diff --git a/crates/github/src/domain/service/sync.rs b/crates/github/src/domain/service/sync.rs
index 4fdb79403d..b355400e1c 100644
--- a/crates/github/src/domain/service/sync.rs
+++ b/crates/github/src/domain/service/sync.rs
@@ -230,6 +230,11 @@ impl<
github_key: github_key.as_ref().to_string(),
owner: owner.to_string(),
repo: repo.to_string(),
+ repository_id: event
+ .payload
+ .get("repository")
+ .and_then(|repository| repository.get("id"))
+ .and_then(|value| value.as_u64()),
number,
url,
display_name: format!("{owner}/{repo}#{number}"),
@@ -260,6 +265,24 @@ impl<
comments: None,
checks: None,
participant_github_user_ids: Self::participant_ids_from_payload(pull_request),
+ draft: pull_request
+ .and_then(|pr| pr.get("draft"))
+ .and_then(|value| value.as_bool()),
+ requested_reviewer_github_user_ids: pull_request
+ .and_then(|pr| pr.get("requested_reviewers"))
+ .and_then(|value| value.as_array())
+ .map(|users| {
+ users
+ .iter()
+ .filter_map(|user| user.get("id").and_then(|value| value.as_u64()))
+ .map(|id| id.to_string())
+ .collect()
+ }),
+ github_updated_at: pull_request
+ .and_then(|pr| pr.get("updated_at"))
+ .and_then(|value| value.as_str())
+ .and_then(|value| chrono::DateTime::parse_from_rfc3339(value).ok())
+ .map(|updated_at| updated_at.with_timezone(&chrono::Utc)),
})
}
@@ -348,6 +371,7 @@ impl<
github_key: fallback.github_key,
owner: fallback.owner,
repo: fallback.repo,
+ repository_id: details.repository_id.or(fallback.repository_id),
number: fallback.number,
url: fallback.url,
display_name: fallback.display_name,
@@ -363,6 +387,11 @@ impl<
participant_github_user_ids: details
.participant_github_user_ids
.or(fallback.participant_github_user_ids),
+ draft: details.draft.or(fallback.draft),
+ requested_reviewer_github_user_ids: details
+ .requested_reviewer_github_user_ids
+ .or(fallback.requested_reviewer_github_user_ids),
+ github_updated_at: details.github_updated_at.or(fallback.github_updated_at),
}
}
diff --git a/crates/github/src/domain/service/sync/test.rs b/crates/github/src/domain/service/sync/test.rs
index 87e074ebf5..c4fd19e1c0 100644
--- a/crates/github/src/domain/service/sync/test.rs
+++ b/crates/github/src/domain/service/sync/test.rs
@@ -33,7 +33,10 @@ use foreign_entity::domain::{
},
ports::{ForeignEntityListQuery, ForeignEntityService},
};
-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;
+use github_pull_requests::domain::{
+ models::GithubPullRequestRow, ports::GithubPullRequestRepository,
+ service::GithubPullRequestServiceImpl,
+};
use macro_user_id::user_id::MacroUserIdStr;
use model::document::{DocumentBasic, DocumentMetadata};
use model_entity::Entity;
@@ -1177,7 +1180,7 @@ type TestSyncServiceImpl = GithubSyncServiceImpl<
StubDocumentService,
StubSyncRepo,
StubSyncClient,
- GithubPullRequestServiceImpl<StubForeignEntityService>,
+ TestPullRequestService,
StubNotificationIngress,
StubRealtime,
>;
@@ -1204,11 +1207,37 @@ impl std::ops::DerefMut for TestGithubSyncService {
type TestServiceWithForeignEntityService = (TestGithubSyncService, StubForeignEntityService);
+type TestPullRequestService =
+ GithubPullRequestServiceImpl<StubForeignEntityService, NoPullRequestRows>;
+
+struct NoPullRequestRows;
+
+impl GithubPullRequestRepository for NoPullRequestRows {
+ type Err = std::convert::Infallible;
+
+ async fn github_key_for(
+ &self,
+ _repository_id: i64,
+ _number: i64,
+ ) -> Result<Option<String>, Self::Err> {
+ Ok(None)
+ }
+
+ async fn upsert_row(&self, _row: &GithubPullRequestRow) -> Result<(), Self::Err> {
+ Ok(())
+ }
+
+ async fn rename_row(&self, _from: &str, _to: &str) -> Result<(), Self::Err> {
+ Ok(())
+ }
+}
+
fn pull_request_service(
foreign_entity_service: &StubForeignEntityService,
-) -> Arc<GithubPullRequestServiceImpl<StubForeignEntityService>> {
+) -> Arc<TestPullRequestService> {
Arc::new(GithubPullRequestServiceImpl::new(
foreign_entity_service.clone(),
+ NoPullRequestRows,
))
}
@@ -1303,6 +1332,7 @@ fn expected_pull_request_metadata(
github_key: "my-org/my-repo/pull/42".to_string(),
owner: "my-org".to_string(),
repo: "my-repo".to_string(),
+ repository_id: None,
number: 42,
url: "https://github.com/my-org/my-repo/pull/42".to_string(),
display_name: "my-org/my-repo#42".to_string(),
@@ -1316,6 +1346,9 @@ fn expected_pull_request_metadata(
comments: None,
checks: None,
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
})
.unwrap()
}
@@ -1480,6 +1513,7 @@ fn backfilled_pull_request(title: &str) -> EnrichedGithubPullRequest {
github_key: "my-org/my-repo/pull/42".to_string(),
owner: "my-org".to_string(),
repo: "my-repo".to_string(),
+ repository_id: None,
number: 42,
url: "https://github.com/my-org/my-repo/pull/42".to_string(),
display_name: "my-org/my-repo#42".to_string(),
@@ -1493,6 +1527,9 @@ fn backfilled_pull_request(title: &str) -> EnrichedGithubPullRequest {
comments: None,
checks: None,
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}
}
@@ -1503,6 +1540,7 @@ fn expected_pull_request_metadata_from_details(
github_key: "my-org/my-repo/pull/42".to_string(),
owner: "my-org".to_string(),
repo: "my-repo".to_string(),
+ repository_id: None,
number: 42,
url: "https://github.com/my-org/my-repo/pull/42".to_string(),
display_name: "my-org/my-repo#42".to_string(),
@@ -1516,6 +1554,9 @@ fn expected_pull_request_metadata_from_details(
comments: details.comments.clone(),
checks: details.checks.clone(),
participant_github_user_ids: details.participant_github_user_ids.clone(),
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
})
.unwrap()
}
@@ -1563,6 +1604,7 @@ fn pull_request_details(
GithubPullRequestDetails {
title: title.to_string(),
state: "open".to_string(),
+ repository_id: None,
merged_at: None,
additions,
deletions,
@@ -1572,6 +1614,9 @@ fn pull_request_details(
comments,
checks,
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}
}
diff --git a/crates/github/src/inbound.rs b/crates/github/src/inbound.rs
index fbd008912d..d882de9aee 100644
--- a/crates/github/src/inbound.rs
+++ b/crates/github/src/inbound.rs
@@ -2,6 +2,8 @@
#[cfg(all(feature = "axum", feature = "sync"))]
pub mod github_sync_router;
+#[cfg(all(feature = "axum", feature = "sync"))]
+pub mod pull_request_index_router;
#[cfg(feature = "axum")]
impl axum::response::IntoResponse for crate::domain::models::GithubError {
diff --git a/crates/github/src/inbound/pull_request_index_router.rs b/crates/github/src/inbound/pull_request_index_router.rs
new file mode 100644
index 0000000000..4e137b18a8
--- /dev/null
+++ b/crates/github/src/inbound/pull_request_index_router.rs
@@ -0,0 +1,88 @@
+//! Internal route that writes typed rows for stored GitHub pull requests, one page of
+//! installations per call.
+//!
+//! Run it once per environment after deploying, then again whenever rows need rebuilding. Each
+//! call is safe to repeat: rows are rewritten from their records.
+//!
+//! ```sh
+//! after=null
+//! while :; do
+//! page=$(curl -sf -X POST "$DSS_URL/internal/github/index-pull-requests" \
+//! -H "x-internal-auth-key: $INTERNAL_API_KEY" \
+//! -H 'content-type: application/json' \
+//! -d "{\"after\": $after}")
+//! echo "$page"
+//! after=$(echo "$page" | jq '.nextAfter')
+//! [ "$after" = "null" ] && break
+//! done
+//! ```
+
+use std::sync::Arc;
+
+use axum::{
+ Json, Router,
+ extract::{FromRef, State},
+ routing::post,
+};
+use macro_authorization::{
+ InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,
+};
+
+use crate::domain::{
+ models::{GithubError, PullRequestIndexPage, PullRequestIndexRequest},
+ ports::GithubPullRequestIndex,
+};
+
+/// State for the pull request index route.
+pub struct PullRequestIndexRouterState<S, Auth> {
+ /// The index service.
+ pub service: Arc<S>,
+ /// Authorization state used to require an internal caller.
+ pub authorization_state: MacroAuthorizationState<Auth>,
+}
+
+impl<S, Auth> Clone for PullRequestIndexRouterState<S, Auth> {
+ fn clone(&self) -> Self {
+ Self {
+ service: self.service.clone(),
+ authorization_state: self.authorization_state.clone(),
+ }
+ }
+}
+
+impl<S, Auth> FromRef<PullRequestIndexRouterState<S, Auth>> for MacroAuthorizationState<Auth> {
+ fn from_ref(state: &PullRequestIndexRouterState<S, Auth>) -> Self {
+ state.authorization_state.clone()
+ }
+}
+
+/// Build the internal router exposing `POST /index-pull-requests`.
+pub fn pull_request_index_router<S, Auth, T>(
+ state: PullRequestIndexRouterState<S, Auth>,
+) -> Router<T>
+where
+ S: GithubPullRequestIndex,
+ Auth: MacroAuthorizationService,
+ T: Send + Sync + 'static,
+{
+ Router::new()
+ .route(
+ "/index-pull-requests",
+ post(index_pull_requests_handler::<S, Auth>),
+ )
+ .with_state(state)
+}
+
+/// Process the next page of installations.
+#[tracing::instrument(skip_all, err)]
+pub async fn index_pull_requests_handler<S, Auth>(
+ State(state): State<PullRequestIndexRouterState<S, Auth>>,
+ _internal: MacroAuthorizationExtractor<Auth, InternalOnly>,
+ Json(request): Json<PullRequestIndexRequest>,
+) -> Result<Json<PullRequestIndexPage>, GithubError>
+where
+ S: GithubPullRequestIndex,
+ Auth: MacroAuthorizationService,
+{
+ Ok(Json(state.service.index_pull_requests(request).await?))
+}
diff --git a/crates/github/src/outbound/pg_github_sync_repo.rs b/crates/github/src/outbound/pg_github_sync_repo.rs
index 2434828cbe..1917c0f0bc 100644
--- a/crates/github/src/outbound/pg_github_sync_repo.rs
+++ b/crates/github/src/outbound/pg_github_sync_repo.rs
@@ -13,7 +13,7 @@ use crate::domain::{
GithubAppInstallationSource, GithubKey, MacroTaskId, ResolvedTeamTaskReference,
TeamTaskReference,
},
- ports::GithubSyncRepo,
+ ports::{GithubInstallationLister, GithubSyncRepo},
};
/// PostgreSQL-backed github repository.
@@ -29,6 +29,31 @@ impl PgGithubSyncRepo {
}
}
+impl GithubInstallationLister for PgGithubSyncRepo {
+ type Err = sqlx::Error;
+
+ #[tracing::instrument(skip(self), err)]
+ async fn list_installation_ids(
+ &self,
+ after: Option<&str>,
+ limit: u32,
+ ) -> Result<Vec<String>, Self::Err> {
+ sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT id AS "id!"
+ FROM github_app_installation
+ WHERE $1::text IS NULL OR id > $1
+ ORDER BY id
+ LIMIT $2
+ "#,
+ after,
+ i64::from(limit),
+ )
+ .fetch_all(&self.pool)
+ .await
+ }
+}
+
impl GithubSyncRepo for PgGithubSyncRepo {
type Err = sqlx::Error;
diff --git a/crates/github/src/outbound/pull_request_metadata.rs b/crates/github/src/outbound/pull_request_metadata.rs
index 64713d4616..f87c50df84 100644
--- a/crates/github/src/outbound/pull_request_metadata.rs
+++ b/crates/github/src/outbound/pull_request_metadata.rs
@@ -50,8 +50,10 @@ pub(crate) async fn fetch_pull_request_metadata(
.as_ref()
.and_then(|user| user.login.clone());
let author_id = pull_request.user.as_ref().and_then(|user| user.id);
+ let requested_reviewer_github_user_ids = user_ids(&pull_request.requested_reviewers);
Ok(GithubPullRequestDetails {
+ repository_id: pull_request.base.repository_id(),
title: pull_request.title,
state: pull_request.state,
merged_at: pull_request.merged_at,
@@ -63,6 +65,9 @@ pub(crate) async fn fetch_pull_request_metadata(
comments,
checks,
participant_github_user_ids,
+ draft: pull_request.draft,
+ requested_reviewer_github_user_ids: Some(requested_reviewer_github_user_ids),
+ github_updated_at: pull_request.updated_at,
})
}
@@ -151,6 +156,7 @@ struct GithubInstallationRepositoriesResponse {
#[derive(Debug, serde::Deserialize)]
struct GithubInstallationRepositoryResponse {
+ id: u64,
name: String,
owner: GithubRepositoryOwnerResponse,
html_url: String,
@@ -168,6 +174,7 @@ impl GithubInstallationRepositoryResponse {
html_url: self.html_url,
default_branch: self.default_branch,
private: self.private,
+ id: self.id,
}
}
}
@@ -183,6 +190,12 @@ struct GithubOpenPullRequestResponse {
title: String,
html_url: String,
body: Option<String>,
+ #[serde(default)]
+ base: GithubPullRequestBaseResponse,
+ #[serde(default)]
+ draft: Option<bool>,
+ #[serde(default)]
+ updated_at: Option<chrono::DateTime<chrono::Utc>>,
user: Option<GithubUserResponse>,
#[serde(default)]
requested_reviewers: Vec<GithubUserResponse>,
@@ -199,6 +212,12 @@ struct GithubPullRequestResponse {
deletions: u64,
body: Option<String>,
head: GithubPullRequestHeadResponse,
+ #[serde(default)]
+ base: GithubPullRequestBaseResponse,
+ #[serde(default)]
+ draft: Option<bool>,
+ #[serde(default)]
+ updated_at: Option<chrono::DateTime<chrono::Utc>>,
user: Option<GithubUserResponse>,
#[serde(default)]
requested_reviewers: Vec<GithubUserResponse>,
@@ -235,6 +254,31 @@ struct GithubPullRequestHeadResponse {
sha: String,
}
+#[derive(Debug, Default, serde::Deserialize)]
+struct GithubPullRequestBaseResponse {
+ repo: Option<GithubRepositoryIdResponse>,
+}
+
+impl GithubPullRequestBaseResponse {
+ fn repository_id(&self) -> Option<u64> {
+ self.repo.as_ref().map(|repo| repo.id)
+ }
+}
+
+#[derive(Debug, serde::Deserialize)]
+struct GithubRepositoryIdResponse {
+ id: u64,
+}
+
+/// The stable numeric ids, as strings, of the users that have one.
+fn user_ids(users: &[GithubUserResponse]) -> Vec<String> {
+ users
+ .iter()
+ .filter_map(|user| user.id)
+ .map(|id| id.to_string())
+ .collect()
+}
+
#[derive(Debug, serde::Deserialize)]
struct GithubUserResponse {
login: Option<String>,
@@ -597,6 +641,7 @@ impl GithubOpenPullRequestResponse {
github_key: GithubKey::new(owner, repo, self.number).to_string(),
owner: owner.to_string(),
repo: repo.to_string(),
+ repository_id: self.base.repository_id(),
number: self.number,
url: self.html_url,
display_name: format!("{owner}/{repo}#{}", self.number),
@@ -611,6 +656,9 @@ impl GithubOpenPullRequestResponse {
checks: None,
participant_github_user_ids: (!participant_ids.is_empty())
.then(|| participant_ids.iter().map(u64::to_string).collect()),
+ draft: self.draft,
+ requested_reviewer_github_user_ids: Some(user_ids(&self.requested_reviewers)),
+ github_updated_at: self.updated_at,
}
}
}
diff --git a/crates/github_pull_requests/Cargo.toml b/crates/github_pull_requests/Cargo.toml
index 5798457bc1..22755bdde9 100644
--- a/crates/github_pull_requests/Cargo.toml
+++ b/crates/github_pull_requests/Cargo.toml
@@ -5,22 +5,26 @@ publish = false
version = "0.1.0"
[features]
-default = ["ports", "service"]
+default = ["outbound", "ports", "service"]
+outbound = ["dep:sqlx", "ports"]
ports = []
schema = ["dep:utoipa"]
service = ["foreign_entity/ports", "ports"]
[dependencies]
+anyhow = { workspace = true }
chrono = { workspace = true }
foreign_entity = { path = "../foreign_entity", default-features = false }
serde = { workspace = true }
serde_json = { workspace = true }
+sqlx = { workspace = true, optional = true }
thiserror = { workspace = true }
tracing = { workspace = true }
utoipa = { workspace = true, optional = true }
+uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
entity_access = { path = "../entity_access", default-features = false, features = ["ports"] }
+macro_db_migrator = { path = "../macro_db_migrator" }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
-uuid = { workspace = true }
diff --git a/crates/github_pull_requests/src/domain/models.rs b/crates/github_pull_requests/src/domain/models.rs
index 8ca24bf8cf..7a383c7dc0 100644
--- a/crates/github_pull_requests/src/domain/models.rs
+++ b/crates/github_pull_requests/src/domain/models.rs
@@ -6,6 +6,7 @@ mod test;
mod key;
mod pull_request;
+use chrono::{DateTime, Utc};
use foreign_entity::domain::models::{ForeignEntity, ForeignEntityError, SourceId};
pub use key::GithubKey;
@@ -36,6 +37,80 @@ pub struct UpsertedGithubPullRequest {
pub participant_github_user_ids: Vec<String>,
}
+/// A GitHub repository's stable id together with its current name.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct GithubRepositoryIdentity {
+ /// The numeric GitHub repository id.
+ pub id: u64,
+ /// The account the repository lives under.
+ pub owner: String,
+ /// The repository name, without its owner.
+ pub name: String,
+}
+
+/// A pull request's typed columns, read from its records' metadata.
+#[derive(Debug, Clone, PartialEq)]
+pub struct GithubPullRequestRow {
+ /// The pull request's `owner/repo/pull/number` key.
+ pub github_key: String,
+ /// The numeric GitHub repository id. `None` when the metadata does not carry one, in which
+ /// case an existing row keeps the id it has.
+ pub repository_id: Option<i64>,
+ /// The pull request number within its repository.
+ pub number: i64,
+ /// The repository owner the pull request was last synced under.
+ pub owner: String,
+ /// The repository name the pull request was last synced under.
+ pub repo: String,
+ /// The pull request title.
+ pub title: Option<String>,
+ /// The normalized pull request status.
+ pub status: Option<GithubPullRequestStatus>,
+ /// Whether the pull request is a draft.
+ pub draft: bool,
+ /// Stable numeric GitHub user id of the author.
+ pub author_github_user_id: Option<String>,
+ /// The author's GitHub login when the pull request was last synced.
+ pub author_login: Option<String>,
+ /// Stable numeric GitHub user ids of the users asked to review.
+ pub requested_reviewer_github_user_ids: Vec<String>,
+ /// Stable numeric GitHub user ids of everyone involved.
+ pub participant_github_user_ids: Vec<String>,
+ /// When GitHub last updated the pull request.
+ pub github_updated_at: Option<DateTime<Utc>>,
+}
+
+impl GithubPullRequestRow {
+ /// Read the typed columns from a pull request record's metadata. `None` when the metadata is
+ /// not pull request metadata.
+ pub fn from_metadata(metadata: &serde_json::Value) -> Option<Self> {
+ let pull_request: EnrichedGithubPullRequest =
+ serde_json::from_value(metadata.clone()).ok()?;
+
+ Some(Self {
+ repository_id: pull_request
+ .repository_id
+ .and_then(|id| i64::try_from(id).ok()),
+ number: i64::try_from(pull_request.number).ok()?,
+ github_key: pull_request.github_key,
+ owner: pull_request.owner,
+ repo: pull_request.repo,
+ title: pull_request.name,
+ status: pull_request.status,
+ draft: pull_request.draft.unwrap_or(false),
+ author_github_user_id: pull_request.author_id.map(|id| id.to_string()),
+ author_login: pull_request.author_login,
+ requested_reviewer_github_user_ids: pull_request
+ .requested_reviewer_github_user_ids
+ .unwrap_or_default(),
+ participant_github_user_ids: pull_request
+ .participant_github_user_ids
+ .unwrap_or_default(),
+ github_updated_at: pull_request.github_updated_at,
+ })
+ }
+}
+
/// Errors from storing GitHub pull requests.
#[derive(Debug, thiserror::Error)]
pub enum GithubPullRequestError {
@@ -45,4 +120,7 @@ pub enum GithubPullRequestError {
/// The foreign entity store failed.
#[error(transparent)]
ForeignEntity(#[from] ForeignEntityError),
+ /// The pull request row store failed.
+ #[error("pull request row storage failed: {0}")]
+ Repository(anyhow::Error),
}
diff --git a/crates/github_pull_requests/src/domain/models/pull_request.rs b/crates/github_pull_requests/src/domain/models/pull_request.rs
index e85966d27f..06ddd2af7a 100644
--- a/crates/github_pull_requests/src/domain/models/pull_request.rs
+++ b/crates/github_pull_requests/src/domain/models/pull_request.rs
@@ -164,6 +164,10 @@ pub struct GithubPullRequestDetails {
pub title: String,
/// The raw GitHub pull request state, usually `open` or `closed`.
pub state: String,
+ /// The numeric id of the base repository, when available.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub repository_id: Option<u64>,
/// The merge timestamp returned by GitHub, when the pull request was merged.
pub merged_at: Option<chrono::DateTime<chrono::Utc>>,
/// The number of added lines reported by GitHub.
@@ -201,6 +205,22 @@ pub struct GithubPullRequestDetails {
deserialize_with = "deserialize_optional_array"
)]
pub participant_github_user_ids: Option<Vec<String>>,
+ /// Whether the pull request is a draft, when available.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub draft: Option<bool>,
+ /// Stable numeric GitHub user ids (as strings) of the users asked to review, when available.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(
+ default,
+ skip_serializing_if = "Option::is_none",
+ deserialize_with = "deserialize_optional_array"
+ )]
+ pub requested_reviewer_github_user_ids: Option<Vec<String>>,
+ /// When GitHub last updated the pull request, when available.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub github_updated_at: Option<chrono::DateTime<chrono::Utc>>,
}
impl GithubPullRequestDetails {
@@ -221,6 +241,11 @@ pub struct EnrichedGithubPullRequest {
pub owner: String,
/// The GitHub repository name.
pub repo: String,
+ /// The numeric GitHub repository id, when known. Unlike `owner` and `repo`, it survives
+ /// renames and transfers.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub repository_id: Option<u64>,
/// The GitHub pull request number.
pub number: u64,
/// The public GitHub URL for the pull request.
@@ -255,6 +280,18 @@ pub struct EnrichedGithubPullRequest {
/// union rather than replacing it (partial write paths must not drop known participants).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub participant_github_user_ids: Option<Vec<String>>,
+ /// Whether the pull request is a draft, when known.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub draft: Option<bool>,
+ /// Stable numeric GitHub user ids (as strings) of the users asked to review, when known.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub requested_reviewer_github_user_ids: Option<Vec<String>>,
+ /// When GitHub last updated the pull request, when known.
+ #[cfg_attr(feature = "schema", schema(ignore))]
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub github_updated_at: Option<chrono::DateTime<chrono::Utc>>,
}
impl EnrichedGithubPullRequest {
@@ -264,6 +301,7 @@ impl EnrichedGithubPullRequest {
github_key: reference.github_key,
owner: reference.owner,
repo: reference.repo,
+ repository_id: None,
number: reference.number,
url: reference.url,
display_name: reference.display_name,
@@ -277,6 +315,9 @@ impl EnrichedGithubPullRequest {
comments: None,
checks: None,
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}
}
@@ -291,6 +332,7 @@ impl EnrichedGithubPullRequest {
github_key: reference.github_key,
owner: reference.owner,
repo: reference.repo,
+ repository_id: details.repository_id,
number: reference.number,
url: reference.url,
display_name: reference.display_name,
@@ -304,6 +346,9 @@ impl EnrichedGithubPullRequest {
comments: details.comments,
checks: details.checks,
participant_github_user_ids: details.participant_github_user_ids,
+ draft: details.draft,
+ requested_reviewer_github_user_ids: details.requested_reviewer_github_user_ids,
+ github_updated_at: details.github_updated_at,
}
}
@@ -311,8 +356,9 @@ impl EnrichedGithubPullRequest {
///
/// Partial refreshes may omit `comments` or `checks`. When an omitted field exists as an array in
/// `existing_metadata`, the existing array is copied forward so richer metadata is not discarded.
- /// The same applies to the scalar `authorLogin`, `authorId`, and `description` fields, which
- /// fallback write paths (such as comment webhooks without a `pull_request` payload) omit.
+ /// The same applies to `authorLogin`, `authorId`, `description`, `repositoryId`, `draft`,
+ /// `requestedReviewerGithubUserIds`, and `githubUpdatedAt`, which fallback write paths (such
+ /// as comment webhooks without a `pull_request` payload) omit.
pub fn foreign_entity_metadata(
&self,
existing_metadata: Option<&serde_json::Value>,
@@ -337,7 +383,15 @@ impl EnrichedGithubPullRequest {
}
}
- for field in ["authorLogin", "authorId", "description"] {
+ for field in [
+ "authorLogin",
+ "authorId",
+ "description",
+ "repositoryId",
+ "draft",
+ "requestedReviewerGithubUserIds",
+ "githubUpdatedAt",
+ ] {
if metadata_object.contains_key(field) {
continue;
}
diff --git a/crates/github_pull_requests/src/domain/models/test.rs b/crates/github_pull_requests/src/domain/models/test.rs
index e7be9e3d45..447dc19f1d 100644
--- a/crates/github_pull_requests/src/domain/models/test.rs
+++ b/crates/github_pull_requests/src/domain/models/test.rs
@@ -25,6 +25,7 @@ fn pull_request_details(
GithubPullRequestDetails {
title: "Add pull request enrichment".to_string(),
state: state.to_string(),
+ repository_id: None,
merged_at,
additions: 42,
deletions: 12,
@@ -34,6 +35,9 @@ fn pull_request_details(
comments: None,
checks: None,
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}
}
@@ -152,6 +156,7 @@ fn pull_request_response_serializes_with_camel_case_fields() {
github_key: reference.github_key,
owner: reference.owner,
repo: reference.repo,
+ repository_id: None,
number: reference.number,
url: reference.url,
display_name: reference.display_name,
@@ -165,6 +170,9 @@ fn pull_request_response_serializes_with_camel_case_fields() {
comments: Some(vec![pull_request_comment()]),
checks: Some(vec![pull_request_check_run()]),
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}],
};
@@ -285,6 +293,7 @@ fn pull_request_enrichment_copies_details_fields() {
let details = GithubPullRequestDetails {
title: "Add pull request enrichment".to_string(),
state: "closed".to_string(),
+ repository_id: None,
merged_at: Some(utc_datetime("2026-05-25T18:54:21Z")),
additions: 42,
deletions: 12,
@@ -294,6 +303,9 @@ fn pull_request_enrichment_copies_details_fields() {
comments: Some(comments.clone()),
checks: Some(checks.clone()),
participant_github_user_ids: Some(vec!["42".to_string(), "583231".to_string()]),
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
};
let enriched = EnrichedGithubPullRequest::from_details(reference.clone(), details);
@@ -336,6 +348,7 @@ fn pull_request_foreign_entity_metadata_serializes_enriched_pull_request() {
let details = GithubPullRequestDetails {
title: "Add pull request enrichment".to_string(),
state: "closed".to_string(),
+ repository_id: None,
merged_at: Some(utc_datetime("2026-05-25T18:54:21Z")),
additions: 42,
deletions: 12,
@@ -345,6 +358,9 @@ fn pull_request_foreign_entity_metadata_serializes_enriched_pull_request() {
comments: Some(vec![pull_request_comment()]),
checks: Some(vec![pull_request_check_run()]),
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
};
let enriched = EnrichedGithubPullRequest::from_details(reference, details);
@@ -457,6 +473,7 @@ fn pull_request_foreign_entity_metadata_keeps_fresh_arrays() {
let details = GithubPullRequestDetails {
title: "Add pull request enrichment".to_string(),
state: "open".to_string(),
+ repository_id: None,
merged_at: None,
additions: 42,
deletions: 12,
@@ -466,6 +483,9 @@ fn pull_request_foreign_entity_metadata_keeps_fresh_arrays() {
comments: Some(comments.clone()),
checks: Some(checks.clone()),
participant_github_user_ids: None,
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
};
let enriched = EnrichedGithubPullRequest::from_details(pull_request_reference(), details);
let existing_metadata = serde_json::json!({
diff --git a/crates/github_pull_requests/src/domain/ports.rs b/crates/github_pull_requests/src/domain/ports.rs
index 28a1e25ad1..4db7f2ada2 100644
--- a/crates/github_pull_requests/src/domain/ports.rs
+++ b/crates/github_pull_requests/src/domain/ports.rs
@@ -5,8 +5,8 @@ use std::future::Future;
use foreign_entity::domain::models::ForeignEntity;
use super::models::{
- EnrichedGithubPullRequest, GithubPullRequestError, UpsertGithubPullRequest,
- UpsertedGithubPullRequest,
+ EnrichedGithubPullRequest, GithubPullRequestError, GithubPullRequestRow,
+ GithubRepositoryIdentity, UpsertGithubPullRequest, UpsertedGithubPullRequest,
};
/// Stores GitHub pull requests as foreign entity records, one record per user or team the pull
@@ -14,6 +14,9 @@ use super::models::{
pub trait GithubPullRequestService: Send + Sync + 'static {
/// Store a pull request's latest data on the record for
/// [`UpsertGithubPullRequest::stored_for`], creating it when that source has none.
+ ///
+ /// A pull request whose repository was renamed or transferred keeps its records: they are
+ /// found by repository id and number and moved to the new key.
fn upsert_pull_request(
&self,
upsert: UpsertGithubPullRequest,
@@ -29,3 +32,55 @@ pub trait GithubPullRequestService: Send + Sync + 'static {
pull_request: &EnrichedGithubPullRequest,
) -> impl Future<Output = Result<Vec<ForeignEntity>, GithubPullRequestError>> + Send;
}
+
+/// Writes typed rows for pull requests stored before rows existed.
+pub trait GithubPullRequestIndexer: Send + Sync + 'static {
+ /// Write rows for the stored pull requests of each of `repositories`, matched by owner and
+ /// name. Returns how many rows were written.
+ fn index_repositories(
+ &self,
+ repositories: &[GithubRepositoryIdentity],
+ ) -> impl Future<Output = Result<u64, GithubPullRequestError>> + Send;
+}
+
+/// Persists the typed columns of pull requests.
+pub trait GithubPullRequestRepository: Send + Sync + 'static {
+ /// Error type returned by repository operations.
+ type Err: Into<anyhow::Error> + Send + std::fmt::Debug;
+
+ /// The key of pull request `number` in the repository with id `repository_id`.
+ fn github_key_for(
+ &self,
+ repository_id: i64,
+ number: i64,
+ ) -> impl Future<Output = Result<Option<String>, Self::Err>> + Send;
+
+ /// Store a pull request's typed columns. A row without a repository id keeps the one it
+ /// has.
+ fn upsert_row(
+ &self,
+ row: &GithubPullRequestRow,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
+ /// Move the row stored under `from` to `to`. When `to` already has a row, the row under
+ /// `from` is removed instead.
+ fn rename_row(
+ &self,
+ from: &str,
+ to: &str,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+}
+
+/// Reads the stored pull request records the indexer writes rows from.
+pub trait GithubPullRequestIndexRepository: Send + Sync + 'static {
+ /// Error type returned by repository operations.
+ type Err: Into<anyhow::Error> + Send + std::fmt::Debug;
+
+ /// The metadata of the most recently updated record of each pull request stored under the
+ /// repository `owner/name`, compared case-insensitively.
+ fn latest_pull_request_metadata(
+ &self,
+ owner: &str,
+ name: &str,
+ ) -> impl Future<Output = Result<Vec<serde_json::Value>, Self::Err>> + Send;
+}
diff --git a/crates/github_pull_requests/src/domain/service.rs b/crates/github_pull_requests/src/domain/service.rs
index 1070b7c710..4f2f454f95 100644
--- a/crates/github_pull_requests/src/domain/service.rs
+++ b/crates/github_pull_requests/src/domain/service.rs
@@ -11,22 +11,28 @@ use foreign_entity::domain::{
use super::{
models::{
EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,
- GithubPullRequestError, GithubPullRequestStatus, UpsertGithubPullRequest,
- UpsertedGithubPullRequest,
+ GithubPullRequestError, GithubPullRequestRow, GithubPullRequestStatus,
+ GithubRepositoryIdentity, UpsertGithubPullRequest, UpsertedGithubPullRequest,
+ },
+ ports::{
+ GithubPullRequestIndexRepository, GithubPullRequestIndexer, GithubPullRequestRepository,
+ GithubPullRequestService,
},
- ports::GithubPullRequestService,
};
-/// Stores pull requests through the foreign entity service.
-pub struct GithubPullRequestServiceImpl<F> {
+/// Stores pull requests through the foreign entity service, with their typed columns in `repo`.
+pub struct GithubPullRequestServiceImpl<F, R> {
foreign_entity_service: F,
+ repo: R,
}
-impl<F: ForeignEntityService> GithubPullRequestServiceImpl<F> {
- /// Create a service that stores pull requests through `foreign_entity_service`.
- pub fn new(foreign_entity_service: F) -> Self {
+impl<F: ForeignEntityService, R: GithubPullRequestRepository> GithubPullRequestServiceImpl<F, R> {
+ /// Create a service that stores pull requests through `foreign_entity_service` and their
+ /// typed columns in `repo`.
+ pub fn new(foreign_entity_service: F, repo: R) -> Self {
Self {
foreign_entity_service,
+ repo,
}
}
@@ -60,9 +66,77 @@ impl<F: ForeignEntityService> GithubPullRequestServiceImpl<F> {
)
.await?)
}
+ /// Move a pull request stored under the name its repository had before a rename or
+ /// transfer to its current key: every record stored for it, and its row.
+ async fn follow_rename(
+ &self,
+ pull_request: &EnrichedGithubPullRequest,
+ ) -> Result<(), GithubPullRequestError> {
+ let (Some(repository_id), Ok(number)) = (
+ pull_request
+ .repository_id
+ .and_then(|id| i64::try_from(id).ok()),
+ i64::try_from(pull_request.number),
+ ) else {
+ return Ok(());
+ };
+ let Some(previous_key) = self
+ .repo
+ .github_key_for(repository_id, number)
+ .await
+ .map_err(repository_error)?
+ else {
+ return Ok(());
+ };
+ if previous_key == pull_request.github_key {
+ return Ok(());
+ }
+
+ for record in self.stored_records(&previous_key).await? {
+ self.foreign_entity_service
+ .patch_foreign_entity(
+ record.id,
+ PatchForeignEntity {
+ foreign_entity_id: Some(pull_request.github_key.clone()),
+ ..PatchForeignEntity::default()
+ },
+ )
+ .await?;
+ }
+ self.repo
+ .rename_row(&previous_key, &pull_request.github_key)
+ .await
+ .map_err(repository_error)
+ }
+
+ /// Write a pull request's typed columns from a record's metadata. Rows are derived data, so
+ /// a failure is logged rather than failing the write of the record itself.
+ async fn store_row(&self, metadata: &serde_json::Value, repository_id: Option<i64>) -> bool {
+ let Some(mut row) = GithubPullRequestRow::from_metadata(metadata) else {
+ tracing::warn!("pull request metadata has no typed columns");
+ return false;
+ };
+ if repository_id.is_some() {
+ row.repository_id = repository_id;
+ }
+
+ match self.repo.upsert_row(&row).await {
+ Ok(()) => true,
+ Err(error) => {
+ tracing::error!(
+ error=?error,
+ github_key=%row.github_key,
+ "failed to store pull request row"
+ );
+ false
+ }
+ }
+ }
}
-impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServiceImpl<F> {
+impl<F: ForeignEntityService, R: GithubPullRequestRepository> GithubPullRequestService
+ for GithubPullRequestServiceImpl<F, R>
+{
#[tracing::instrument(
err,
skip(self, upsert),
@@ -76,6 +150,7 @@ impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServ
pull_request,
stored_for,
} = upsert;
+ self.follow_rename(&pull_request).await?;
let records = self.stored_records(&pull_request.github_key).await?;
let existing = records.iter().find(|record| {
@@ -115,6 +190,7 @@ impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServ
.await?
}
};
+ self.store_row(&foreign_entity.metadata, None).await;
Ok(UpsertedGithubPullRequest {
foreign_entity,
@@ -147,13 +223,56 @@ impl<F: ForeignEntityService> GithubPullRequestService for GithubPullRequestServ
}
}
}
- match first_error {
- Some(error) => Err(error),
- None => Ok(refreshed),
+ // A successful subset can omit metadata held only by a failed record. Preserve the
+ // existing typed row rather than replacing it with that incomplete projection.
+ if let Some(error) = first_error {
+ return Err(error);
+ }
+ if let Some(latest) = refreshed.iter().max_by_key(|record| record.updated_at) {
+ self.store_row(&latest.metadata, None).await;
+ }
+ Ok(refreshed)
+ }
+}
+
+impl<F, R> GithubPullRequestIndexer for GithubPullRequestServiceImpl<F, R>
+where
+ F: ForeignEntityService,
+ R: GithubPullRequestRepository + GithubPullRequestIndexRepository,
+{
+ #[tracing::instrument(err, skip(self, repositories), fields(repositories = repositories.len()))]
+ async fn index_repositories(
+ &self,
+ repositories: &[GithubRepositoryIdentity],
+ ) -> Result<u64, GithubPullRequestError> {
+ let mut indexed = 0;
+ for repository in repositories {
+ let Some(repository_id) = i64::try_from(repository.id).ok().filter(|id| *id != 0)
+ else {
+ continue;
+ };
+ let pull_requests = GithubPullRequestIndexRepository::latest_pull_request_metadata(
+ &self.repo,
+ &repository.owner,
+ &repository.name,
+ )
+ .await
+ .map_err(repository_error)?;
+ for metadata in &pull_requests {
+ if self.store_row(metadata, Some(repository_id)).await {
+ indexed += 1;
+ }
+ }
}
+
+ Ok(indexed)
}
}
+fn repository_error(error: impl Into<anyhow::Error>) -> GithubPullRequestError {
+ GithubPullRequestError::Repository(error.into())
+}
+
fn status_from_metadata(metadata: &serde_json::Value) -> Option<GithubPullRequestStatus> {
metadata
.get("status")
diff --git a/crates/github_pull_requests/src/domain/service/test.rs b/crates/github_pull_requests/src/domain/service/test.rs
index bdbb540a7e..677b9b5e4a 100644
--- a/crates/github_pull_requests/src/domain/service/test.rs
+++ b/crates/github_pull_requests/src/domain/service/test.rs
@@ -1,3 +1,4 @@
+use std::convert::Infallible;
use std::sync::{Arc, Mutex};
use chrono::Utc;
@@ -13,10 +14,13 @@ use uuid::Uuid;
use super::GithubPullRequestServiceImpl;
use crate::domain::{
models::{
- EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,
- GithubPullRequestStatus, UpsertGithubPullRequest,
+ EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, GithubPullRequestRow,
+ GithubPullRequestStatus, GithubRepositoryIdentity, UpsertGithubPullRequest,
+ },
+ ports::{
+ GithubPullRequestIndexRepository, GithubPullRequestIndexer, GithubPullRequestRepository,
+ GithubPullRequestService,
},
- ports::GithubPullRequestService,
};
const GITHUB_KEY: &str = "macro/app/pull/7";
@@ -140,6 +144,9 @@ impl ForeignEntityService for StubForeignEntityService {
.iter_mut()
.find(|record| record.id == id)
.ok_or(ForeignEntityError::NotFound(id))?;
+ if let Some(foreign_entity_id) = patch.foreign_entity_id {
+ record.foreign_entity_id = foreign_entity_id;
+ }
if let Some(metadata) = patch.metadata {
record.metadata = metadata;
}
@@ -148,11 +155,94 @@ impl ForeignEntityService for StubForeignEntityService {
}
}
+#[derive(Clone, Default)]
+struct StubPullRequestRows {
+ rows: Arc<Mutex<Vec<GithubPullRequestRow>>>,
+ /// Stored pull request metadata by repository owner and name.
+ stored: Arc<Mutex<Vec<(String, String, serde_json::Value)>>>,
+}
+
+impl StubPullRequestRows {
+ fn rows(&self) -> Vec<GithubPullRequestRow> {
+ self.rows.lock().unwrap().clone()
+ }
+}
+
+impl GithubPullRequestRepository for StubPullRequestRows {
+ type Err = Infallible;
+
+ async fn github_key_for(
+ &self,
+ repository_id: i64,
+ number: i64,
+ ) -> Result<Option<String>, Self::Err> {
+ Ok(self
+ .rows()
+ .into_iter()
+ .find(|row| row.repository_id == Some(repository_id) && row.number == number)
+ .map(|row| row.github_key))
+ }
+
+ async fn upsert_row(&self, row: &GithubPullRequestRow) -> Result<(), Self::Err> {
+ let mut rows = self.rows.lock().unwrap();
+ match rows
+ .iter_mut()
+ .find(|existing| existing.github_key == row.github_key)
+ {
+ Some(existing) => {
+ let repository_id = row.repository_id.or(existing.repository_id);
+ *existing = GithubPullRequestRow {
+ repository_id,
+ ..row.clone()
+ };
+ }
+ None => rows.push(row.clone()),
+ }
+ Ok(())
+ }
+
+ async fn rename_row(&self, from: &str, to: &str) -> Result<(), Self::Err> {
+ let mut rows = self.rows.lock().unwrap();
+ if rows.iter().any(|row| row.github_key == to) {
+ rows.retain(|row| row.github_key != from);
+ } else if let Some(row) = rows.iter_mut().find(|row| row.github_key == from) {
+ row.github_key = to.to_owned();
+ }
+ Ok(())
+ }
+}
+
+impl GithubPullRequestIndexRepository for StubPullRequestRows {
+ type Err = Infallible;
+
+ async fn latest_pull_request_metadata(
+ &self,
+ owner: &str,
+ name: &str,
+ ) -> Result<Vec<serde_json::Value>, Self::Err> {
+ Ok(self
+ .stored
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|(stored_owner, stored_name, _)| stored_owner == owner && stored_name == name)
+ .map(|(_, _, metadata)| metadata.clone())
+ .collect())
+ }
+}
+
+type TestService = GithubPullRequestServiceImpl<StubForeignEntityService, StubPullRequestRows>;
+
+fn service(foreign_entities: &StubForeignEntityService, rows: &StubPullRequestRows) -> TestService {
+ GithubPullRequestServiceImpl::new(foreign_entities.clone(), rows.clone())
+}
+
fn pull_request(status: GithubPullRequestStatus) -> EnrichedGithubPullRequest {
EnrichedGithubPullRequest {
github_key: GITHUB_KEY.to_string(),
owner: "macro".to_string(),
repo: "app".to_string(),
+ repository_id: None,
number: 7,
url: "https://github.com/macro/app/pull/7".to_string(),
display_name: "macro/app#7".to_string(),
@@ -166,6 +256,9 @@ fn pull_request(status: GithubPullRequestStatus) -> EnrichedGithubPullRequest {
comments: None,
checks: None,
participant_github_user_ids: Some(vec!["42".to_string()]),
+ draft: None,
+ requested_reviewer_github_user_ids: None,
+ github_updated_at: None,
}
}
@@ -194,7 +287,8 @@ fn user() -> SourceId {
#[tokio::test]
async fn upsert_creates_the_record_for_a_source_without_one() {
let foreign_entities = StubForeignEntityService::default();
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
let upserted = service
.upsert_pull_request(UpsertGithubPullRequest {
@@ -231,7 +325,8 @@ async fn upsert_merges_into_the_sources_own_record() {
let user_record = stored_record(&user(), serde_json::json!({ "status": "closed" }));
let foreign_entities =
StubForeignEntityService::with_records(vec![user_record.clone(), team_record.clone()]);
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
let upserted = service
.upsert_pull_request(UpsertGithubPullRequest {
@@ -271,7 +366,8 @@ async fn a_sources_first_record_starts_from_another_sources_metadata() {
}),
);
let foreign_entities = StubForeignEntityService::with_records(vec![team_record]);
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
let upserted = service
.upsert_pull_request(UpsertGithubPullRequest {
@@ -296,7 +392,8 @@ async fn refresh_updates_every_stored_record() {
let user_record = stored_record(&user(), serde_json::json!({ "status": "open" }));
let foreign_entities =
StubForeignEntityService::with_records(vec![team_record.clone(), user_record.clone()]);
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
let refreshed = service
.refresh_pull_request(&pull_request(GithubPullRequestStatus::Closed))
@@ -318,7 +415,8 @@ async fn refresh_updates_every_stored_record() {
#[tokio::test]
async fn refresh_without_stored_records_does_nothing() {
let foreign_entities = StubForeignEntityService::default();
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
let refreshed = service
.refresh_pull_request(&pull_request(GithubPullRequestStatus::Open))
@@ -336,7 +434,13 @@ async fn refresh_attempts_later_records_after_a_patch_failure() {
let foreign_entities =
StubForeignEntityService::with_records(vec![team_record.clone(), user_record.clone()]);
foreign_entities.state.lock().unwrap().failing_patches = vec![team_record.id];
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let original_metadata = pull_request(GithubPullRequestStatus::Open)
+ .foreign_entity_metadata(None)
+ .unwrap();
+ let original_row = GithubPullRequestRow::from_metadata(&original_metadata).unwrap();
+ rows.rows.lock().unwrap().push(original_row.clone());
+ let service = service(&foreign_entities, &rows);
let result = service
.refresh_pull_request(&pull_request(GithubPullRequestStatus::Closed))
@@ -360,6 +464,7 @@ async fn refresh_attempts_later_records_after_a_patch_failure() {
assert_eq!(records[0].metadata["status"], "open");
assert_eq!(records[1].metadata["status"], "closed");
assert!(foreign_entities.creates().is_empty());
+ assert_eq!(rows.rows(), vec![original_row]);
}
#[tokio::test]
@@ -371,7 +476,8 @@ async fn refresh_attempts_every_record_when_all_patches_fail() {
let ids = records.iter().map(|record| record.id).collect::<Vec<_>>();
let foreign_entities = StubForeignEntityService::with_records(records.clone());
foreign_entities.state.lock().unwrap().failing_patches = ids.clone();
- let service = GithubPullRequestServiceImpl::new(foreign_entities.clone());
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
assert!(
service
@@ -388,4 +494,148 @@ async fn refresh_attempts_every_record_when_all_patches_fail() {
ids
);
assert_eq!(foreign_entities.records(), records);
+ assert!(rows.rows().is_empty());
+}
+
+#[tokio::test]
+async fn upsert_writes_the_row_from_the_merged_metadata() {
+ let foreign_entities = StubForeignEntityService::default();
+ let rows = StubPullRequestRows::default();
+ let service = service(&foreign_entities, &rows);
+
+ service
+ .upsert_pull_request(UpsertGithubPullRequest {
+ pull_request: EnrichedGithubPullRequest {
+ repository_id: Some(99),
+ draft: Some(true),
+ requested_reviewer_github_user_ids: Some(vec!["8".to_string()]),
+ ..pull_request(GithubPullRequestStatus::Open)
+ },
+ stored_for: user(),
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(
+ rows.rows(),
+ vec![GithubPullRequestRow {
+ github_key: GITHUB_KEY.to_string(),
+ repository_id: Some(99),
+ number: 7,
+ owner: "macro".to_string(),
+ repo: "app".to_string(),
+ title: Some("Add pull request storage".to_string()),
+ status: Some(GithubPullRequestStatus::Open),
+ draft: true,
+ author_github_user_id: None,
+ author_login: None,
+ requested_reviewer_github_user_ids: vec!["8".to_string()],
+ participant_github_user_ids: vec!["42".to_string()],
+ github_updated_at: None,
+ }]
+ );
+}
+
+#[tokio::test]
+async fn upsert_after_a_repository_rename_moves_every_record_and_the_row() {
+ let old_metadata = serde_json::to_value(EnrichedGithubPullRequest {
+ repository_id: Some(99),
+ ..pull_request(GithubPullRequestStatus::Open)
+ })
+ .unwrap();
+ let team_record = stored_record(&team(), old_metadata.clone());
+ let user_record = stored_record(&user(), old_metadata.clone());
+ let foreign_entities =
+ StubForeignEntityService::with_records(vec![team_record.clone(), user_record.clone()]);
+ let rows = StubPullRequestRows::default();
+ rows.rows
+ .lock()
+ .unwrap()
+ .push(GithubPullRequestRow::from_metadata(&old_metadata).unwrap());
+ let service = service(&foreign_entities, &rows);
+
+ let upserted = service
+ .upsert_pull_request(UpsertGithubPullRequest {
+ pull_request: EnrichedGithubPullRequest {
+ github_key: "macro/renamed/pull/7".to_string(),
+ repo: "renamed".to_string(),
+ url: "https://github.com/macro/renamed/pull/7".to_string(),
+ display_name: "macro/renamed#7".to_string(),
+ repository_id: Some(99),
+ ..pull_request(GithubPullRequestStatus::Merged)
+ },
+ stored_for: team(),
+ })
+ .await
+ .unwrap();
+
+ assert!(foreign_entities.creates().is_empty());
+ assert_eq!(upserted.foreign_entity.id, team_record.id);
+ assert_eq!(
+ upserted.previous_status,
+ Some(GithubPullRequestStatus::Open)
+ );
+ assert!(
+ foreign_entities
+ .records()
+ .iter()
+ .all(|record| record.foreign_entity_id == "macro/renamed/pull/7")
+ );
+ let stored_rows = rows.rows();
+ assert_eq!(stored_rows.len(), 1);
+ assert_eq!(stored_rows[0].github_key, "macro/renamed/pull/7");
+ assert_eq!(stored_rows[0].repo, "renamed");
+ assert_eq!(stored_rows[0].status, Some(GithubPullRequestStatus::Merged));
+}
+
+#[tokio::test]
+async fn index_writes_a_row_for_each_stored_pull_request_of_each_known_repository() {
+ let metadata = |key: &str, number: u64, status: &str| {
+ serde_json::json!({
+ "githubKey": key,
+ "owner": "Macro",
+ "repo": "App",
+ "number": number,
+ "url": format!("https://github.com/{key}"),
+ "displayName": key,
+ "status": status,
+ })
+ };
+ let rows = StubPullRequestRows::default();
+ rows.stored.lock().unwrap().extend([
+ (
+ "macro".to_string(),
+ "app".to_string(),
+ metadata("Macro/App/pull/7", 7, "closed"),
+ ),
+ (
+ "macro".to_string(),
+ "app-web".to_string(),
+ metadata("macro/app-web/pull/1", 1, "open"),
+ ),
+ ]);
+ let service = service(&StubForeignEntityService::default(), &rows);
+
+ let indexed = service
+ .index_repositories(&[
+ GithubRepositoryIdentity {
+ id: 99,
+ owner: "macro".to_string(),
+ name: "app".to_string(),
+ },
+ GithubRepositoryIdentity {
+ id: 0,
+ owner: "macro".to_string(),
+ name: "app-web".to_string(),
+ },
+ ])
+ .await
+ .unwrap();
+
+ assert_eq!(indexed, 1);
+ let stored_rows = rows.rows();
+ assert_eq!(stored_rows.len(), 1);
+ assert_eq!(stored_rows[0].github_key, "Macro/App/pull/7");
+ assert_eq!(stored_rows[0].repository_id, Some(99));
+ assert_eq!(stored_rows[0].status, Some(GithubPullRequestStatus::Closed));
}
diff --git a/crates/github_pull_requests/src/lib.rs b/crates/github_pull_requests/src/lib.rs
index fee5934598..ceb938ee0e 100644
--- a/crates/github_pull_requests/src/lib.rs
+++ b/crates/github_pull_requests/src/lib.rs
@@ -1,12 +1,17 @@
//! GitHub pull requests stored as foreign entity records.
//!
//! Owns the pull request model, the rules for merging refreshed pull request data into what is
-//! already stored, and the service that writes pull request records.
+//! already stored, the typed `github_pull_request` columns read from that data, and the service
+//! that writes pull request records.
//!
//! # Architecture
//!
//! - **domain**: Contains domain models, ports, and service implementation.
+//! - **outbound**: Contains adapters for external persistence systems.
#![deny(missing_docs)]
pub mod domain;
+
+#[cfg(feature = "outbound")]
+pub mod outbound;
diff --git a/crates/github_pull_requests/src/outbound.rs b/crates/github_pull_requests/src/outbound.rs
new file mode 100644
index 0000000000..9cdfba4f20
--- /dev/null
+++ b/crates/github_pull_requests/src/outbound.rs
@@ -0,0 +1,3 @@
+//! Outbound adapters for GitHub pull request storage.
+
+pub mod pg_github_pull_request_repo;
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs
new file mode 100644
index 0000000000..5651ca1bd3
--- /dev/null
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs
@@ -0,0 +1,124 @@
+//! PostgreSQL storage for the typed columns of GitHub pull requests.
+
+mod index;
+#[cfg(test)]
+mod test;
+
+use sqlx::PgPool;
+
+use crate::domain::{models::GithubPullRequestRow, ports::GithubPullRequestRepository};
+
+/// Stores pull request rows in the `github_pull_request` table.
+#[derive(Clone)]
+pub struct PgGithubPullRequestRepo {
+ pool: PgPool,
+}
+
+impl PgGithubPullRequestRepo {
+ /// Create a repository over `pool`.
+ pub fn new(pool: PgPool) -> Self {
+ Self { pool }
+ }
+}
+
+impl GithubPullRequestRepository for PgGithubPullRequestRepo {
+ type Err = sqlx::Error;
+
+ #[tracing::instrument(err, skip(self))]
+ async fn github_key_for(
+ &self,
+ repository_id: i64,
+ number: i64,
+ ) -> Result<Option<String>, Self::Err> {
+ sqlx::query_scalar!(
+ r#"
+ SELECT github_key
+ FROM github_pull_request
+ WHERE repository_id = $1 AND number = $2
+ "#,
+ repository_id,
+ number,
+ )
+ .fetch_optional(&self.pool)
+ .await
+ }
+
+ #[tracing::instrument(err, skip(self, row), fields(github_key = %row.github_key))]
+ async fn upsert_row(&self, row: &GithubPullRequestRow) -> Result<(), Self::Err> {
+ sqlx::query!(
+ r#"
+ INSERT INTO github_pull_request (
+ github_key,
+ repository_id,
+ number,
+ owner,
+ repo,
+ title,
+ status,
+ draft,
+ author_github_user_id,
+ author_login,
+ requested_reviewer_github_user_ids,
+ participant_github_user_ids,
+ github_updated_at
+ )
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+ ON CONFLICT (github_key) DO UPDATE SET
+ repository_id = COALESCE(EXCLUDED.repository_id, github_pull_request.repository_id),
+ number = EXCLUDED.number,
+ owner = EXCLUDED.owner,
+ repo = EXCLUDED.repo,
+ title = EXCLUDED.title,
+ status = EXCLUDED.status,
+ draft = EXCLUDED.draft,
+ author_github_user_id = EXCLUDED.author_github_user_id,
+ author_login = EXCLUDED.author_login,
+ requested_reviewer_github_user_ids = EXCLUDED.requested_reviewer_github_user_ids,
+ participant_github_user_ids = EXCLUDED.participant_github_user_ids,
+ github_updated_at = EXCLUDED.github_updated_at,
+ updated_at = NOW()
+ "#,
+ row.github_key,
+ row.repository_id,
+ row.number,
+ row.owner,
+ row.repo,
+ row.title.as_deref(),
+ row.status.map(|status| status.as_str()),
+ row.draft,
+ row.author_github_user_id.as_deref(),
+ row.author_login.as_deref(),
+ &row.requested_reviewer_github_user_ids,
+ &row.participant_github_user_ids,
+ row.github_updated_at,
+ )
+ .execute(&self.pool)
+ .await?;
+
+ Ok(())
+ }
+
+ #[tracing::instrument(err, skip(self))]
+ async fn rename_row(&self, from: &str, to: &str) -> Result<(), Self::Err> {
+ sqlx::query!(
+ r#"
+ WITH taken AS (
+ SELECT 1 FROM github_pull_request WHERE github_key = $2
+ ),
+ removed AS (
+ DELETE FROM github_pull_request
+ WHERE github_key = $1 AND EXISTS (SELECT 1 FROM taken)
+ )
+ UPDATE github_pull_request
+ SET github_key = $2, updated_at = NOW()
+ WHERE github_key = $1 AND NOT EXISTS (SELECT 1 FROM taken)
+ "#,
+ from,
+ to,
+ )
+ .execute(&self.pool)
+ .await?;
+
+ Ok(())
+ }
+}
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs
new file mode 100644
index 0000000000..ce6119bf2b
--- /dev/null
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs
@@ -0,0 +1,36 @@
+//! The stored pull request records the indexer writes rows from.
+
+#[cfg(test)]
+mod test;
+
+use super::PgGithubPullRequestRepo;
+use crate::domain::{
+ models::GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, ports::GithubPullRequestIndexRepository,
+};
+
+/// The records belong to the foreign_entity crate; the indexer only reads them.
+impl GithubPullRequestIndexRepository for PgGithubPullRequestRepo {
+ type Err = sqlx::Error;
+
+ #[tracing::instrument(err, skip(self))]
+ async fn latest_pull_request_metadata(
+ &self,
+ owner: &str,
+ name: &str,
+ ) -> Result<Vec<serde_json::Value>, Self::Err> {
+ let key_prefix = format!("{owner}/{name}/pull/");
+ sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT ON (fe.foreign_entity_id) fe.metadata AS "metadata!: serde_json::Value"
+ FROM foreign_entity fe
+ WHERE fe.foreign_entity_source = $1::text
+ AND left(lower(fe.foreign_entity_id), length($2::text)) = lower($2::text)
+ ORDER BY fe.foreign_entity_id, fe.updated_at DESC, fe.id DESC
+ "#,
+ GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,
+ key_prefix,
+ )
+ .fetch_all(&self.pool)
+ .await
+ }
+}
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs
new file mode 100644
index 0000000000..4d78918985
--- /dev/null
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs
@@ -0,0 +1,129 @@
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use serde_json::{Value, json};
+use sqlx::PgPool;
+
+use super::super::PgGithubPullRequestRepo;
+use crate::domain::ports::GithubPullRequestIndexRepository;
+
+async fn insert_record(
+ pool: &PgPool,
+ key: &str,
+ source: &str,
+ stored_for_id: &str,
+ metadata: Value,
+ minutes_ago: i32,
+) {
+ sqlx::query(
+ r#"
+ INSERT INTO foreign_entity (
+ id, foreign_entity_id, foreign_entity_source, metadata, stored_for_id,
+ stored_for_auth_entity, updated_at
+ )
+ VALUES (gen_random_uuid(), $1, $2, $3, $4, 'user', NOW() - make_interval(mins => $5::int))
+ "#,
+ )
+ .bind(key)
+ .bind(source)
+ .bind(metadata)
+ .bind(stored_for_id)
+ .bind(minutes_ago)
+ .execute(pool)
+ .await
+ .expect("record should be inserted");
+}
+
+fn sorted(mut metadata: Vec<Value>) -> Vec<Value> {
+ metadata.sort_by_key(|value| value.to_string());
+ metadata
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn latest_metadata_covers_each_pull_request_in_the_repository_once(pool: PgPool) {
+ let user = "macro|[email protected]";
+ let team = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
+ insert_record(
+ &pool,
+ "Macro-Inc/App/pull/1",
+ "github_pull_request",
+ user,
+ json!({ "n": 1 }),
+ 0,
+ )
+ .await;
+ insert_record(
+ &pool,
+ "macro-inc/app/pull/2",
+ "github_pull_request",
+ user,
+ json!({ "n": "older" }),
+ 10,
+ )
+ .await;
+ insert_record(
+ &pool,
+ "macro-inc/app/pull/2",
+ "github_pull_request",
+ team,
+ json!({ "n": "newer" }),
+ 1,
+ )
+ .await;
+ insert_record(
+ &pool,
+ "macro-inc/app-web/pull/3",
+ "github_pull_request",
+ user,
+ json!({ "n": 3 }),
+ 0,
+ )
+ .await;
+ insert_record(
+ &pool,
+ "macro-inc/app/pull/4",
+ "linear_issue",
+ user,
+ json!({ "n": 4 }),
+ 0,
+ )
+ .await;
+
+ let metadata = PgGithubPullRequestRepo::new(pool)
+ .latest_pull_request_metadata("macro-inc", "app")
+ .await
+ .expect("records should load");
+
+ assert_eq!(
+ sorted(metadata),
+ sorted(vec![json!({ "n": 1 }), json!({ "n": "newer" })])
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn repository_names_match_literally(pool: PgPool) {
+ let user = "macro|[email protected]";
+ insert_record(
+ &pool,
+ "macro-inc/appXrepo/pull/1",
+ "github_pull_request",
+ user,
+ json!({ "n": 1 }),
+ 0,
+ )
+ .await;
+ insert_record(
+ &pool,
+ "macro-inc/app_repo/pull/2",
+ "github_pull_request",
+ user,
+ json!({ "n": 2 }),
+ 0,
+ )
+ .await;
+
+ let metadata = PgGithubPullRequestRepo::new(pool)
+ .latest_pull_request_metadata("macro-inc", "app_repo")
+ .await
+ .expect("records should load");
+
+ assert_eq!(metadata, vec![json!({ "n": 2 })]);
+}
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/test.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/test.rs
new file mode 100644
index 0000000000..e53a9e57b1
--- /dev/null
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/test.rs
@@ -0,0 +1,122 @@
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::PgPool;
+
+use super::PgGithubPullRequestRepo;
+use crate::domain::{
+ models::{GithubPullRequestRow, GithubPullRequestStatus},
+ ports::GithubPullRequestRepository,
+};
+
+async fn stored_row(
+ pool: &PgPool,
+ github_key: &str,
+) -> Option<(Option<i64>, String, Option<String>)> {
+ sqlx::query_as(
+ r#"
+ SELECT repository_id, repo, status
+ FROM github_pull_request
+ WHERE github_key = $1
+ "#,
+ )
+ .bind(github_key)
+ .fetch_optional(pool)
+ .await
+ .expect("row lookup should succeed")
+}
+
+fn row(github_key: &str, repository_id: Option<i64>) -> GithubPullRequestRow {
+ GithubPullRequestRow {
+ github_key: github_key.to_string(),
+ repository_id,
+ number: 7,
+ owner: "macro".to_string(),
+ repo: "app".to_string(),
+ title: Some("Add pull request storage".to_string()),
+ status: Some(GithubPullRequestStatus::Open),
+ draft: false,
+ author_github_user_id: Some("42".to_string()),
+ author_login: Some("octocat".to_string()),
+ requested_reviewer_github_user_ids: vec!["8".to_string()],
+ participant_github_user_ids: vec!["8".to_string(), "42".to_string()],
+ github_updated_at: None,
+ }
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn upsert_stores_a_row_found_by_repository_and_number(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+
+ repo.upsert_row(&row("macro/app/pull/7", Some(99)))
+ .await
+ .expect("upsert should succeed");
+
+ assert_eq!(
+ repo.github_key_for(99, 7).await.unwrap(),
+ Some("macro/app/pull/7".to_string())
+ );
+ assert_eq!(repo.github_key_for(99, 8).await.unwrap(), None);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn upsert_without_a_repository_id_keeps_the_one_the_row_has(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+
+ repo.upsert_row(&row("macro/app/pull/7", None))
+ .await
+ .expect("upsert should succeed");
+ assert_eq!(
+ stored_row(&pool, "macro/app/pull/7").await,
+ Some((None, "app".to_string(), Some("open".to_string())))
+ );
+
+ repo.upsert_row(&row("macro/app/pull/7", Some(99)))
+ .await
+ .expect("upsert should succeed");
+ repo.upsert_row(&GithubPullRequestRow {
+ status: Some(GithubPullRequestStatus::Merged),
+ ..row("macro/app/pull/7", None)
+ })
+ .await
+ .expect("upsert should succeed");
+
+ assert_eq!(
+ stored_row(&pool, "macro/app/pull/7").await,
+ Some((Some(99), "app".to_string(), Some("merged".to_string())))
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn rename_moves_the_row_or_drops_it_when_the_new_key_has_one(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ repo.upsert_row(&row("macro/app/pull/7", Some(99)))
+ .await
+ .unwrap();
+
+ repo.rename_row("macro/app/pull/7", "macro/renamed/pull/7")
+ .await
+ .expect("rename should succeed");
+ assert_eq!(stored_row(&pool, "macro/app/pull/7").await, None);
+ assert_eq!(
+ repo.github_key_for(99, 7).await.unwrap(),
+ Some("macro/renamed/pull/7".to_string())
+ );
+
+ repo.upsert_row(&GithubPullRequestRow {
+ number: 8,
+ ..row("macro/app/pull/8", Some(99))
+ })
+ .await
+ .unwrap();
+ repo.upsert_row(&GithubPullRequestRow {
+ number: 9,
+ ..row("macro/renamed/pull/8", None)
+ })
+ .await
+ .unwrap();
+ repo.rename_row("macro/app/pull/8", "macro/renamed/pull/8")
+ .await
+ .expect("rename onto an existing row should succeed");
+
+ assert_eq!(stored_row(&pool, "macro/app/pull/8").await, None);
+ assert!(stored_row(&pool, "macro/renamed/pull/8").await.is_some());
+}
diff --git a/crates/macro_db_client/migrations/20260926173005_github_pull_request.sql b/crates/macro_db_client/migrations/20260926173005_github_pull_request.sql
new file mode 100644
index 0000000000..d3cada85c0
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260926173005_github_pull_request.sql
@@ -0,0 +1,32 @@
+-- Typed columns for GitHub pull requests, so pull requests can be filtered and sorted without
+-- reading JSON metadata. One row per pull request, shared by every foreign entity record stored
+-- for it: a record joins its row on its owner/repo/pull/number key. Written by the
+-- github_pull_requests crate from the records' metadata.
+--
+-- The repository's numeric id and the pull request number identify a pull request across
+-- repository renames and transfers. The id is filled in once it is known.
+CREATE TABLE github_pull_request (
+ github_key TEXT PRIMARY KEY,
+ repository_id BIGINT,
+ number BIGINT NOT NULL,
+ owner TEXT NOT NULL,
+ repo TEXT NOT NULL,
+ title TEXT,
+ status TEXT CHECK (status IN ('open', 'closed', 'merged')),
+ draft BOOLEAN NOT NULL DEFAULT FALSE,
+ author_github_user_id TEXT,
+ author_login TEXT,
+ requested_reviewer_github_user_ids TEXT[] NOT NULL DEFAULT '{}',
+ participant_github_user_ids TEXT[] NOT NULL DEFAULT '{}',
+ github_updated_at TIMESTAMPTZ,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ CONSTRAINT uq_github_pull_request_repository_number UNIQUE (repository_id, number)
+);
+
+CREATE INDEX idx_github_pull_request_author ON github_pull_request (author_github_user_id);
+CREATE INDEX idx_github_pull_request_github_updated_at ON github_pull_request (github_updated_at DESC);
+CREATE INDEX idx_github_pull_request_requested_reviewers
+ ON github_pull_request USING GIN (requested_reviewer_github_user_ids);
+CREATE INDEX idx_github_pull_request_participants
+ ON github_pull_request USING GIN (participant_github_user_ids);
diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.toml
index 26c69950b7..de7d6a180c 100644
--- a/services/authentication_service/Cargo.toml
+++ b/services/authentication_service/Cargo.toml
@@ -85,6 +85,7 @@ github = { path = "../../crates/github", default-features = false, features = [
"outbound",
] }
github_pull_requests = { path = "../../crates/github_pull_requests", default-features = false, features = [
+ "outbound",
"service",
] }
http-body-util = { workspace = true }
diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rs
index 0d832129c1..030330c184 100644
--- a/services/authentication_service/src/api/context.rs
+++ b/services/authentication_service/src/api/context.rs
@@ -24,7 +24,10 @@ use github::domain::service::GithubLinkServiceImpl;
use github::outbound::github_auth_client::GithubAuthImpl;
use github::outbound::github_oauth_client::GithubOauthImpl;
use github::outbound::pg_github_repo::PgGithubRepo;
-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;
+use github_pull_requests::{
+ domain::service::GithubPullRequestServiceImpl,
+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
+};
use gtm_invite::{
domain::service::GtmInviteServiceImpl, outbound::pg_gtm_invite_repo::PgGtmInviteRepo,
};
@@ -118,7 +121,10 @@ pub(crate) type GithubLinkServiceType = GithubLinkServiceImpl<
PgGithubRepo,
GithubOauthImpl,
GithubAuthImpl,
- GithubPullRequestServiceImpl<ForeignEntityServiceImpl<PgForeignEntityRepo>>,
+ GithubPullRequestServiceImpl<
+ ForeignEntityServiceImpl<PgForeignEntityRepo>,
+ PgGithubPullRequestRepo,
+ >,
>;
pub(crate) type EntityAccessServiceType = EntityAccessServiceImpl<PgAccessRepository>;
diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rs
index adec62a2f3..c9c7458b04 100644
--- a/services/authentication_service/src/main.rs
+++ b/services/authentication_service/src/main.rs
@@ -32,7 +32,10 @@ use github::{
pg_github_repo::PgGithubRepo,
},
};
-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;
+use github_pull_requests::{
+ domain::service::GithubPullRequestServiceImpl,
+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
+};
use loops_client::LoopsClient;
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{
@@ -454,7 +457,10 @@ async fn main() -> anyhow::Result<()> {
PgGithubRepo::new(db.clone()),
GithubOauthImpl::default(),
GithubAuthImpl::new(auth_client.clone(), redis_multiplexed_conn),
- GithubPullRequestServiceImpl::new(foreign_entity_service),
+ GithubPullRequestServiceImpl::new(
+ foreign_entity_service,
+ PgGithubPullRequestRepo::new(db.clone()),
+ ),
GithubLinkConfig {
client_id: config.github_client_id.to_string(),
client_secret: config.github_client_secret.to_string(),
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index ab41e9e51a..c3a3028561 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -131,6 +131,7 @@ github = { path = "../../crates/github", default-features = false, features = [
"sync",
] }
github_pull_requests = { path = "../../crates/github_pull_requests", default-features = false, features = [
+ "outbound",
"service",
] }
graphql_soup = { path = "../../crates/graphql_soup" }
diff --git a/services/document_storage_service/src/api.rs b/services/document_storage_service/src/api.rs
index 7cbbc6c041..b2b1b2ced0 100644
--- a/services/document_storage_service/src/api.rs
+++ b/services/document_storage_service/src/api.rs
@@ -308,6 +308,12 @@ fn api_router(state: ApiContext) -> Router {
"/internal",
internal::router(state.clone())
.nest("/notifications", notification::router())
+ .nest(
+ "/github",
+ github::inbound::pull_request_index_router::pull_request_index_router(
+ state.github_pull_request_index_state.clone(),
+ ),
+ )
.nest(
"/search",
search_service::search_router()
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index ff03cdf01d..9ac9dcb700 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -87,11 +87,15 @@ use foreign_entity::{
outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
};
use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
-use github::domain::service::GithubSyncServiceImpl;
+use github::domain::service::{GithubSyncServiceImpl, PullRequestIndexService};
+use github::inbound::pull_request_index_router::PullRequestIndexRouterState;
use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;
use github::outbound::github_sync_client::GithubSyncClientImpl;
use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;
-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;
+use github_pull_requests::{
+ domain::service::GithubPullRequestServiceImpl,
+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
+};
use initiative::{
domain::service::InitiativeServiceImpl, inbound::axum_router::InitiativeRouterState,
outbound::PgInitiativeRepo,
@@ -659,16 +663,28 @@ pub(crate) type ForeignEntityServiceType = ForeignEntityServiceImpl<PgForeignEnt
pub(crate) type DssForeignEntityState =
ForeignEntityRouterState<ForeignEntityServiceType, EntityAccessService, AuthorizationService>;
+/// Type alias for the GitHub pull request service.
+pub(crate) type GithubPullRequestServiceType =
+ GithubPullRequestServiceImpl<ForeignEntityServiceType, PgGithubPullRequestRepo>;
+
/// Type alias for the github sync service.
pub(crate) type GithubSyncServiceType = GithubSyncServiceImpl<
DocumentService,
PgGithubSyncRepo,
GithubSyncClientImpl,
- GithubPullRequestServiceImpl<ForeignEntityServiceType>,
+ GithubPullRequestServiceType,
NotificationIngressType,
ConnectionGatewayGithubRealtime,
>;
+/// Type alias for the GitHub pull request index service.
+pub(crate) type GithubPullRequestIndexServiceType =
+ PullRequestIndexService<PgGithubSyncRepo, GithubSyncClientImpl, GithubPullRequestServiceType>;
+
+/// Type alias for the GitHub pull request index router state.
+pub(crate) type DssGithubPullRequestIndexState =
+ PullRequestIndexRouterState<GithubPullRequestIndexServiceType, AuthorizationService>;
+
/// Type alias for the cal.com webhook service.
pub(crate) type CalWebhookServiceType = CalWebhookServiceImpl<AnalyticsClientSink>;
@@ -713,6 +729,7 @@ pub(crate) struct ApiContext {
pub redis_client: Arc<Redis>,
pub s3_client: Arc<S3>,
pub github_sync_service: Arc<GithubSyncServiceType>,
+ pub github_pull_request_index_state: DssGithubPullRequestIndexState,
pub dynamodb_client: Arc<DynamodbClient>,
pub dynamo_db: aws_sdk_dynamodb::Client,
pub soup_router_state: DssSoupState,
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 041b18332b..06cfde2111 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -89,11 +89,17 @@ use foreign_entity::{
outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
};
use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
-use github::domain::service::{GithubSyncConfig, GithubSyncServiceImpl};
+use github::domain::service::{
+ GithubSyncConfig, GithubSyncServiceImpl, InstallationTokenConfig, PullRequestIndexService,
+};
+use github::inbound::pull_request_index_router::PullRequestIndexRouterState;
use github::outbound::connection_gateway_realtime::ConnectionGatewayGithubRealtime;
use github::outbound::github_sync_client::GithubSyncClientImpl;
use github::outbound::pg_github_sync_repo::PgGithubSyncRepo;
-use github_pull_requests::domain::service::GithubPullRequestServiceImpl;
+use github_pull_requests::{
+ domain::service::GithubPullRequestServiceImpl,
+ outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
+};
use graphql_scheduled_action::ScheduledActionGraphqlContext;
use harnesses::outbound::pg_harness_repo::PgHarnessRepo;
use initiative::{
@@ -577,6 +583,7 @@ async fn run() -> anyhow::Result<()> {
document_service.clone(),
Arc::new(GithubPullRequestServiceImpl::new(
ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
)),
(*notification_ingress_service).clone(),
PgGithubSyncRepo::new(db.clone()),
@@ -584,6 +591,22 @@ async fn run() -> anyhow::Result<()> {
ConnectionGatewayGithubRealtime::new(conn_gateway_client.clone()),
);
+ let github_pull_request_index_state = PullRequestIndexRouterState {
+ service: Arc::new(PullRequestIndexService::new(
+ InstallationTokenConfig {
+ client_id: config.github_sync_app_client_id.to_string(),
+ private_key_pem: config.github_sync_app_pem_secret_key.as_ref().to_string(),
+ },
+ PgGithubSyncRepo::new(db.clone()),
+ GithubSyncClientImpl::default(),
+ GithubPullRequestServiceImpl::new(
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ PgGithubPullRequestRepo::new(db.clone()),
+ ),
+ )),
+ authorization_state: authorization_state.clone(),
+ };
+
let foreign_entity_state = ForeignEntityRouterState::new(
foreign_entity_service.clone(),
entity_access_service.clone(),
@@ -1816,6 +1839,7 @@ async fn run() -> anyhow::Result<()> {
)),
graphql_entity_mutation_service,
github_sync_service: Arc::new(github_sync_service_impl),
+ github_pull_request_index_state,
foreign_entity_state,
db: db.clone(),
readonly_db: readonly_pool::ReadOnlyPool(readonly_db.clone()),