Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(github): safely initialize verified pull request index rows (#7438)

dev-rb241077930173
diff --git a/.sqlx/query-07bd38eeebbfbe491683e8847603e5db8ea18a4a31bf629cb5815468978966df.json b/.sqlx/query-07bd38eeebbfbe491683e8847603e5db8ea18a4a31bf629cb5815468978966df.json
new file mode 100644
index 0000000000..d7400e9730
--- /dev/null
+++ b/.sqlx/query-07bd38eeebbfbe491683e8847603e5db8ea18a4a31bf629cb5815468978966df.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT to_jsonb(pr) AS \"snapshot!: serde_json::Value\"\n FROM github_pull_request pr ORDER BY github_key",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "snapshot!: serde_json::Value",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "07bd38eeebbfbe491683e8847603e5db8ea18a4a31bf629cb5815468978966df"
+}
diff --git a/.sqlx/query-4f15555a7c13bf38d91db2d521d803d96a252c73055a096b35ea2ff2bde0523d.json b/.sqlx/query-4f15555a7c13bf38d91db2d521d803d96a252c73055a096b35ea2ff2bde0523d.json
new file mode 100644
index 0000000000..2bc80311a9
--- /dev/null
+++ b/.sqlx/query-4f15555a7c13bf38d91db2d521d803d96a252c73055a096b35ea2ff2bde0523d.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_advisory_xact_lock(hashtextextended(lower($1), 0))",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "pg_advisory_xact_lock",
+ "type_info": "Void"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "4f15555a7c13bf38d91db2d521d803d96a252c73055a096b35ea2ff2bde0523d"
+}
diff --git a/.sqlx/query-7def3a5b4182036221075516893c9d6870a82fbdffe6a9bc93fe37c6fc6ca8e3.json b/.sqlx/query-7def3a5b4182036221075516893c9d6870a82fbdffe6a9bc93fe37c6fc6ca8e3.json
new file mode 100644
index 0000000000..a3d74c2f0f
--- /dev/null
+++ b/.sqlx/query-7def3a5b4182036221075516893c9d6870a82fbdffe6a9bc93fe37c6fc6ca8e3.json
@@ -0,0 +1,48 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT github_key, repository_id, number, owner, repo\n FROM github_pull_request\n WHERE lower(github_key) = lower($1)\n OR (repository_id = $2 AND number = $3)\n ORDER BY github_key\n FOR UPDATE\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "github_key",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "repository_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 2,
+ "name": "number",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 3,
+ "name": "owner",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 4,
+ "name": "repo",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ true,
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "7def3a5b4182036221075516893c9d6870a82fbdffe6a9bc93fe37c6fc6ca8e3"
+}
diff --git a/.sqlx/query-94f5a5246d7c12c3b472a767e8f81a2783496b17d443bc64a0a900b2a0cf1606.json b/.sqlx/query-94f5a5246d7c12c3b472a767e8f81a2783496b17d443bc64a0a900b2a0cf1606.json
new file mode 100644
index 0000000000..abac83a50f
--- /dev/null
+++ b/.sqlx/query-94f5a5246d7c12c3b472a767e8f81a2783496b17d443bc64a0a900b2a0cf1606.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT github_key FROM github_pull_request WHERE lower(github_key) = lower($1) LIMIT 1 FOR UPDATE",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "github_key",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "94f5a5246d7c12c3b472a767e8f81a2783496b17d443bc64a0a900b2a0cf1606"
+}
diff --git a/.sqlx/query-3850e97e55874949d78f9b242819dc9bc1479de6b1d8a3488801dc494eedebe2.json b/.sqlx/query-a74cb158102fb166fe5c32127188fac0f39c57cacf43c677e513996f06be6f85.json
similarity index 88%
rename from .sqlx/query-3850e97e55874949d78f9b242819dc9bc1479de6b1d8a3488801dc494eedebe2.json
rename to .sqlx/query-a74cb158102fb166fe5c32127188fac0f39c57cacf43c677e513996f06be6f85.json
index f6e71ce6df..5073a5d220 100644
--- a/.sqlx/query-3850e97e55874949d78f9b242819dc9bc1479de6b1d8a3488801dc494eedebe2.json
+++ b/.sqlx/query-a74cb158102fb166fe5c32127188fac0f39c57cacf43c677e513996f06be6f85.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "SELECT github_key, repository_id, number, owner, repo, title, status,\n draft, author_github_user_id, author_login,\n requested_reviewer_github_user_ids, participant_github_user_ids,\n github_updated_at, assignees, labels, reviews,\n base_ref, base_sha, head_ref, head_sha\n FROM github_pull_request WHERE github_key = $1 FOR UPDATE",
+ "query": "SELECT github_key, repository_id, number, owner, repo, title, status,\n draft, author_github_user_id, author_login,\n requested_reviewer_github_user_ids, participant_github_user_ids,\n github_updated_at, assignees, labels, reviews,\n base_ref, base_sha, head_ref, head_sha\n FROM github_pull_request\n WHERE github_key = $1\n OR (lower(github_key) = lower($1) AND repository_id = $2 AND number = $3)\n ORDER BY github_key\n FOR UPDATE",
"describe": {
"columns": [
{
@@ -106,7 +106,9 @@
],
"parameters": {
"Left": [
- "Text"
+ "Text",
+ "Int8",
+ "Int8"
]
},
"nullable": [
@@ -132,5 +134,5 @@
true
]
},
- "hash": "3850e97e55874949d78f9b242819dc9bc1479de6b1d8a3488801dc494eedebe2"
+ "hash": "a74cb158102fb166fe5c32127188fac0f39c57cacf43c677e513996f06be6f85"
}
diff --git a/.sqlx/query-aa9592f76b10e53415c3d06dcdc76d1277d5c0d8522b6b0a64b5b71951da2edc.json b/.sqlx/query-aa9592f76b10e53415c3d06dcdc76d1277d5c0d8522b6b0a64b5b71951da2edc.json
new file mode 100644
index 0000000000..8757ce8259
--- /dev/null
+++ b/.sqlx/query-aa9592f76b10e53415c3d06dcdc76d1277d5c0d8522b6b0a64b5b71951da2edc.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO foreign_entity (id, foreign_entity_id, foreign_entity_source, metadata,\n stored_for_id, stored_for_auth_entity)\n SELECT id, 'macro/app/pull/7', 'github_pull_request', '{}'::jsonb, 'user', 'user'\n FROM unnest($1::uuid[]) AS id",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "aa9592f76b10e53415c3d06dcdc76d1277d5c0d8522b6b0a64b5b71951da2edc"
+}
diff --git a/.sqlx/query-b697cca28b0208c99dafa96fbcd5ff017a234dd7b8dc7c479db4b226b34f826d.json b/.sqlx/query-b697cca28b0208c99dafa96fbcd5ff017a234dd7b8dc7c479db4b226b34f826d.json
new file mode 100644
index 0000000000..6a73992401
--- /dev/null
+++ b/.sqlx/query-b697cca28b0208c99dafa96fbcd5ff017a234dd7b8dc7c479db4b226b34f826d.json
@@ -0,0 +1,34 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO github_pull_request (\n github_key, repository_id, number, owner, repo, title, status, draft,\n author_github_user_id, author_login, requested_reviewer_github_user_ids,\n participant_github_user_ids, github_updated_at, assignees, labels, reviews,\n review_decision, base_ref, base_sha, head_ref, head_sha\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,\n $14::jsonb, $15::jsonb, $16::jsonb, $17, $18, $19, $20, $21\n )\n ON CONFLICT DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8",
+ "Int8",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Bool",
+ "Text",
+ "Text",
+ "TextArray",
+ "TextArray",
+ "Timestamptz",
+ "Jsonb",
+ "Jsonb",
+ "Jsonb",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "b697cca28b0208c99dafa96fbcd5ff017a234dd7b8dc7c479db4b226b34f826d"
+}
diff --git a/.sqlx/query-c96b43f28b4c42a7805ce09dcbee2838d33e401e24c9d608cbe6489620da14dd.json b/.sqlx/query-c96b43f28b4c42a7805ce09dcbee2838d33e401e24c9d608cbe6489620da14dd.json
new file mode 100644
index 0000000000..23200b262a
--- /dev/null
+++ b/.sqlx/query-c96b43f28b4c42a7805ce09dcbee2838d33e401e24c9d608cbe6489620da14dd.json
@@ -0,0 +1,49 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT id, foreign_entity_source AS source, foreign_entity_id AS github_key,\n updated_at, metadata\n FROM foreign_entity\n WHERE foreign_entity_source = $1::text\n AND left(lower(foreign_entity_id), length($2::text)) = lower($2::text)\n AND ($3::uuid IS NULL OR id > $3)\n ORDER BY id ASC\n LIMIT $4\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "source",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "github_key",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 3,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 4,
+ "name": "metadata",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Uuid",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "c96b43f28b4c42a7805ce09dcbee2838d33e401e24c9d608cbe6489620da14dd"
+}
diff --git a/.sqlx/query-ce05940abd256a8c92e81ada0b2a4935e9353b7ed741fc398e4d36043df86d8e.json b/.sqlx/query-ce05940abd256a8c92e81ada0b2a4935e9353b7ed741fc398e4d36043df86d8e.json
new file mode 100644
index 0000000000..e858608d1c
--- /dev/null
+++ b/.sqlx/query-ce05940abd256a8c92e81ada0b2a4935e9353b7ed741fc398e4d36043df86d8e.json
@@ -0,0 +1,17 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO foreign_entity (id, foreign_entity_id, foreign_entity_source, metadata,\n stored_for_id, stored_for_auth_entity)\n VALUES ($1, $2, $3, $4, 'macro|[email protected]', 'user')",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Text",
+ "Text",
+ "Jsonb"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "ce05940abd256a8c92e81ada0b2a4935e9353b7ed741fc398e4d36043df86d8e"
+}
diff --git a/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json b/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json
deleted file mode 100644
index e8fb0d829f..0000000000
--- a/.sqlx/query-e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT DISTINCT ON (fe.foreign_entity_id) fe.metadata AS \"metadata!: serde_json::Value\"\n FROM foreign_entity fe\n WHERE fe.foreign_entity_source = $1::text\n AND left(lower(fe.foreign_entity_id), length($2::text)) = lower($2::text)\n ORDER BY fe.foreign_entity_id, fe.updated_at DESC, fe.id DESC\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "metadata!: serde_json::Value",
- "type_info": "Jsonb"
- }
- ],
- "parameters": {
- "Left": [
- "Text",
- "Text"
- ]
- },
- "nullable": [
- false
- ]
- },
- "hash": "e015cc7677f63b6dc09eb419879a0bec4ad127cebb49758e076b4e489da45e22"
-}
diff --git a/crates/github/src/domain/models/repository.rs b/crates/github/src/domain/models/repository.rs
index e5836765d3..e6d9c5e2ee 100644
--- a/crates/github/src/domain/models/repository.rs
+++ b/crates/github/src/domain/models/repository.rs
@@ -47,16 +47,27 @@ pub struct PullRequestIndexRequest {
}
/// What one page of pull request indexing did.
-#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
+#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PullRequestIndexPage {
/// Installations processed in this page, including those that failed.
pub installations: u32,
/// Repositories listed across the page's installations.
pub repositories: u32,
- /// Pull request rows written.
+ /// Typed PR rows newly initialized; existing rows and retries do not count.
pub indexed_pull_requests: u64,
- /// Installations whose repositories GitHub would not list, such as suspended ones.
+ /// Initialization attempts that found a matching existing row without changing it.
+ pub already_indexed_pull_requests: u64,
+ /// Source records skipped because repository identity is absent.
+ pub unverified_records: u64,
+ /// Malformed source records or invalid keys/numbers.
+ pub invalid_records: u64,
+ /// Source records or initialization attempts rejected for conflicting identity.
+ pub identity_conflicts: u64,
+ /// Failed repository listings, source reads, row writes, or invalid repository identities.
+ pub failures: u64,
+ /// Installations with listing/storage failures, malformed records, or identity conflicts.
+ /// Retry the original page request: using a failed ID as `after` skips that installation.
pub failed_installation_ids: Vec<String>,
/// Pass as `after` to continue; absent once every installation has been processed.
pub next_after: Option<String>,
diff --git a/crates/github/src/domain/service/pull_request_index.rs b/crates/github/src/domain/service/pull_request_index.rs
index ae2f2ab476..c5055dd2c5 100644
--- a/crates/github/src/domain/service/pull_request_index.rs
+++ b/crates/github/src/domain/service/pull_request_index.rs
@@ -1,10 +1,8 @@
//! Writing typed rows for pull requests stored before rows existed.
//!
-//! A pull request row is identified by its repository's numeric id, which older records do not
-//! carry. This pages through every installation, asks GitHub which repositories it covers, and
-//! has the pull request store write a row for each record synced under one of those
-//! repositories' names. Rows are rewritten from the record each time, so a page can be re-run
-//! safely.
+//! Pages through installations and initializes missing typed rows from stored, identity-verified
+//! source records. Installation repository IDs never supply absent source identity. Existing
+//! rows remain untouched on retries; malformed records, conflicts, and failures are reported.
use github_pull_requests::domain::{
models::GithubRepositoryIdentity, ports::GithubPullRequestIndexer,
@@ -86,11 +84,8 @@ where
None
};
let mut page = PullRequestIndexPage {
- installations: 0,
- repositories: 0,
- indexed_pull_requests: 0,
- failed_installation_ids: Vec::new(),
next_after,
+ ..Default::default()
};
if installation_ids.is_empty() {
return Ok(page);
@@ -100,19 +95,29 @@ where
for installation_id in installation_ids {
page.installations += 1;
let Some(repositories) = self.list_repositories(&jwt, &installation_id).await else {
+ page.failures += 1;
page.failed_installation_ids.push(installation_id);
continue;
};
page.repositories += u32::try_from(repositories.len()).unwrap_or(u32::MAX);
- page.indexed_pull_requests += self
- .pull_requests
- .index_repositories(&repositories)
- .await
- .map_err(|error| {
- GithubError::Internal(anyhow::anyhow!(
- "could not index pull requests for installation {installation_id}: {error}"
- ))
- })?;
+ match self.pull_requests.index_repositories(&repositories).await {
+ Ok(summary) => {
+ page.indexed_pull_requests += summary.inserted;
+ page.already_indexed_pull_requests += summary.already_present;
+ page.unverified_records += summary.unverified_records;
+ page.invalid_records += summary.invalid_records;
+ page.identity_conflicts += summary.identity_conflicts;
+ page.failures += summary.failures;
+ if summary.has_failures() {
+ page.failed_installation_ids.push(installation_id);
+ }
+ }
+ Err(error) => {
+ page.failures += 1;
+ tracing::error!(error=?error, installation_id, "failed to index installation pull requests");
+ page.failed_installation_ids.push(installation_id);
+ }
+ }
}
Ok(page)
diff --git a/crates/github/src/domain/service/pull_request_index/test.rs b/crates/github/src/domain/service/pull_request_index/test.rs
index ac19880ad7..73afa1b259 100644
--- a/crates/github/src/domain/service/pull_request_index/test.rs
+++ b/crates/github/src/domain/service/pull_request_index/test.rs
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
-use github_pull_requests::domain::models::GithubPullRequestError;
+use github_pull_requests::domain::models::{GithubPullRequestError, PullRequestIndexSummary};
use super::*;
use crate::domain::models::GithubRepository;
@@ -84,6 +84,7 @@ impl GithubRepositoryClient for FakeClient {
struct RecordingPullRequests {
batches: Arc<Mutex<Vec<Vec<GithubRepositoryIdentity>>>>,
fail: bool,
+ summaries: HashMap<u64, PullRequestIndexSummary>,
}
impl RecordingPullRequests {
@@ -96,7 +97,7 @@ impl GithubPullRequestIndexer for RecordingPullRequests {
async fn index_repositories(
&self,
repositories: &[GithubRepositoryIdentity],
- ) -> Result<u64, GithubPullRequestError> {
+ ) -> Result<PullRequestIndexSummary, GithubPullRequestError> {
if self.fail {
return Err(GithubPullRequestError::Repository(anyhow::anyhow!(
"database unavailable"
@@ -106,7 +107,14 @@ impl GithubPullRequestIndexer for RecordingPullRequests {
.lock()
.expect("lock")
.push(repositories.to_vec());
- Ok(repositories.len() as u64)
+ Ok(repositories
+ .first()
+ .and_then(|repository| self.summaries.get(&repository.id))
+ .copied()
+ .unwrap_or(PullRequestIndexSummary {
+ inserted: repositories.len() as u64,
+ ..Default::default()
+ }))
}
}
@@ -163,6 +171,7 @@ async fn pages_through_installations_and_reports_where_to_resume() {
indexed_pull_requests: 2,
failed_installation_ids: Vec::new(),
next_after: Some("2".to_string()),
+ ..Default::default()
}
);
@@ -181,6 +190,7 @@ async fn pages_through_installations_and_reports_where_to_resume() {
indexed_pull_requests: 1,
failed_installation_ids: Vec::new(),
next_after: None,
+ ..Default::default()
}
);
@@ -222,19 +232,20 @@ async fn installations_github_will_not_list_are_reported_and_skipped() {
}
#[tokio::test]
-async fn a_storage_failure_stops_the_page() {
+async fn a_storage_failure_is_reported_without_stopping_later_installations() {
let pull_requests = RecordingPullRequests {
fail: true,
- ..RecordingPullRequests::default()
+ ..Default::default()
};
- let service = service(&["1"], two_installation_client(), pull_requests);
-
- let error = service
+ let service = service(&["1", "3"], two_installation_client(), pull_requests);
+ let page = service
.index_pull_requests(PullRequestIndexRequest::default())
.await
- .expect_err("a storage failure should fail the page");
-
- assert!(matches!(error, GithubError::Internal(_)));
+ .unwrap();
+ assert_eq!(page.installations, 2);
+ assert_eq!(page.failures, 2);
+ assert_eq!(page.failed_installation_ids, vec!["1", "3"]);
+ assert_eq!(page.next_after, None);
}
#[tokio::test]
@@ -256,3 +267,60 @@ async fn a_zero_limit_still_processes_one_installation() {
assert_eq!(page.installations, 1);
assert_eq!(page.next_after, Some("1".to_string()));
}
+
+#[tokio::test]
+async fn partial_index_failures_preserve_counts_and_continue_installations() {
+ let pull_requests = RecordingPullRequests {
+ summaries: HashMap::from([(
+ 100,
+ PullRequestIndexSummary {
+ inserted: 2,
+ already_present: 3,
+ unverified_records: 4,
+ invalid_records: 5,
+ identity_conflicts: 6,
+ failures: 7,
+ },
+ )]),
+ ..Default::default()
+ };
+ let page = service(&["1", "3"], two_installation_client(), pull_requests)
+ .index_pull_requests(PullRequestIndexRequest {
+ limit: Some(2),
+ after: None,
+ })
+ .await
+ .unwrap();
+ assert_eq!(page.indexed_pull_requests, 3);
+ assert_eq!(page.already_indexed_pull_requests, 3);
+ assert_eq!(page.unverified_records, 4);
+ assert_eq!(page.invalid_records, 5);
+ assert_eq!(page.identity_conflicts, 6);
+ assert_eq!(page.failures, 7);
+ assert_eq!(page.failed_installation_ids, vec!["1"]);
+ assert_eq!(page.next_after.as_deref(), Some("3"));
+ let json = serde_json::to_value(page).unwrap();
+ assert_eq!(json["indexedPullRequests"], 3);
+ assert_eq!(json["unverifiedRecords"], 4);
+}
+
+#[tokio::test]
+async fn unverified_skips_do_not_mark_an_installation_failed() {
+ let pull_requests = RecordingPullRequests {
+ summaries: HashMap::from([(
+ 100,
+ PullRequestIndexSummary {
+ unverified_records: 4,
+ ..Default::default()
+ },
+ )]),
+ ..Default::default()
+ };
+ let page = service(&["1"], two_installation_client(), pull_requests)
+ .index_pull_requests(PullRequestIndexRequest::default())
+ .await
+ .unwrap();
+ assert_eq!(page.unverified_records, 4);
+ assert_eq!(page.indexed_pull_requests, 0);
+ assert!(page.failed_installation_ids.is_empty());
+}
diff --git a/crates/github_pull_requests/src/domain/models.rs b/crates/github_pull_requests/src/domain/models.rs
index 661f206610..acd010b40b 100644
--- a/crates/github_pull_requests/src/domain/models.rs
+++ b/crates/github_pull_requests/src/domain/models.rs
@@ -5,6 +5,7 @@ mod test;
mod changes;
mod diff;
+mod index;
mod key;
mod pull_request;
@@ -20,6 +21,7 @@ pub use diff::{
ChangesetRange, GitRef, GithubPullRequestDiff, GithubPullRequestDiffError, PullRequestRef,
RepositorySlug,
};
+pub use index::{PullRequestIndexOutcome, PullRequestIndexRecord, PullRequestIndexSummary};
pub use key::GithubKey;
pub use pull_request::{
EnrichGithubPullRequestsProxyRequest, EnrichGithubPullRequestsResponse,
diff --git a/crates/github_pull_requests/src/domain/models/index.rs b/crates/github_pull_requests/src/domain/models/index.rs
new file mode 100644
index 0000000000..bcfd6f2b41
--- /dev/null
+++ b/crates/github_pull_requests/src/domain/models/index.rs
@@ -0,0 +1,54 @@
+//! Identity-checked, insert-only reconstruction of the pull request index.
+
+use chrono::{DateTime, Utc};
+use uuid::Uuid;
+
+/// One stored source record, before identity validation or sparse-field merging.
+#[derive(Debug, Clone)]
+pub struct PullRequestIndexRecord {
+ /// Stable record ID used to order equal source update times.
+ pub id: Uuid,
+ /// Source namespace of the stored record.
+ pub source: String,
+ /// Association key on the stored record, independent of its metadata.
+ pub github_key: String,
+ /// Source record update time, not evidence of repository identity.
+ pub updated_at: DateTime<Utc>,
+ /// Supplied PR fields, retaining omitted-versus-empty distinctions.
+ pub metadata: serde_json::Value,
+}
+
+/// Result of atomically initializing a verified typed row without updating existing rows.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PullRequestIndexOutcome {
+ /// A previously missing row was inserted.
+ Inserted,
+ /// The same case-insensitive key and stable identity already exist; nothing changed.
+ AlreadyPresent,
+ /// A key or stable identity belongs to a different or unverified row; nothing changed.
+ IdentityConflict,
+}
+
+/// Outcomes across repositories. Expected legacy skips are not storage failures.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct PullRequestIndexSummary {
+ /// Typed PR rows newly inserted, excluding retries and existing rows.
+ pub inserted: u64,
+ /// Initialization attempts that found an existing matching typed row.
+ pub already_present: u64,
+ /// Source records omitted because they do not carry a repository ID.
+ pub unverified_records: u64,
+ /// Malformed source records, invalid keys, or invalid numeric identities.
+ pub invalid_records: u64,
+ /// Source records or initialization attempts rejected for conflicting identity.
+ pub identity_conflicts: u64,
+ /// Repository reads, invalid installation repository identities, or row writes that failed.
+ pub failures: u64,
+}
+
+impl PullRequestIndexSummary {
+ /// Whether the batch needs investigation or retry, excluding expected unverified skips.
+ pub fn has_failures(&self) -> bool {
+ self.failures > 0 || self.identity_conflicts > 0 || self.invalid_records > 0
+ }
+}
diff --git a/crates/github_pull_requests/src/domain/ports.rs b/crates/github_pull_requests/src/domain/ports.rs
index 0bfb1ed802..3fa10eda85 100644
--- a/crates/github_pull_requests/src/domain/ports.rs
+++ b/crates/github_pull_requests/src/domain/ports.rs
@@ -14,7 +14,8 @@ use super::models::{
EnrichedGithubPullRequest, GithubPullRequestChangesError, GithubPullRequestChangeset,
GithubPullRequestDiff, GithubPullRequestDiffError, GithubPullRequestError,
GithubPullRequestFacets, GithubPullRequestRow, GithubPullRequestSortDirection,
- GithubPullRequestWrite, GithubRepositoryIdentity, PullRequestRef, StoredGithubPullRequest,
+ GithubPullRequestWrite, GithubRepositoryIdentity, PullRequestIndexOutcome,
+ PullRequestIndexRecord, PullRequestIndexSummary, PullRequestRef, StoredGithubPullRequest,
UpsertGithubPullRequest, UpsertedGithubPullRequest,
};
@@ -49,14 +50,14 @@ pub trait GithubPullRequestService: Send + Sync + 'static {
) -> impl Future<Output = Result<StoredGithubPullRequest, GithubPullRequestError>> + Send;
}
-/// Writes typed rows for pull requests stored before rows existed.
+/// Initializes missing typed rows from identity-verified source records; never refreshes existing rows.
pub trait GithubPullRequestIndexer: Send + Sync + 'static {
- /// Write rows for the stored pull requests of each of `repositories`, matched by owner and
- /// name. Returns how many rows were written.
+ /// Reconstruct missing rows from all verified source records of `repositories`.
+ /// Missing source identity remains untyped. Independent failures are counted and logged.
fn index_repositories(
&self,
repositories: &[GithubRepositoryIdentity],
- ) -> impl Future<Output = Result<u64, GithubPullRequestError>> + Send;
+ ) -> impl Future<Output = Result<PullRequestIndexSummary, GithubPullRequestError>> + Send;
}
/// Persists the typed columns of pull requests.
@@ -74,6 +75,8 @@ pub trait GithubPullRequestRepository: Send + Sync + 'static {
/// Atomically merge supplied typed columns with the shared row. Omitted fields retain
/// stored values. Concurrent creation and updates for the same key serialize before reading.
/// A provided base/head ref replaces its name and SHA together, including missing components.
+ /// A non-null stored identity cannot be reassigned. A case-only fallback requires the same
+ /// verified identity and retains the stored key; repository renames remain a separate operation.
fn upsert_row(
&self,
row: &GithubPullRequestWrite,
@@ -99,13 +102,23 @@ pub trait GithubPullRequestIndexRepository: Send + Sync + 'static {
/// Error type returned by repository operations.
type Err: Into<anyhow::Error> + Send + std::fmt::Debug;
- /// The metadata of the most recently updated record of each pull request stored under the
- /// repository `owner/name`, compared case-insensitively.
- fn latest_pull_request_metadata(
+ /// All PR source records under `owner/name`, compared case-insensitively and literally.
+ /// Retain older and case-variant records for domain validation and sparse reconstruction.
+ fn pull_request_index_records(
&self,
owner: &str,
name: &str,
- ) -> impl Future<Output = Result<Vec<serde_json::Value>, Self::Err>> + Send;
+ ) -> impl Future<Output = Result<Vec<PullRequestIndexRecord>, Self::Err>> + Send;
+
+ /// Atomically initialize a verified row only if its key and stable identity are absent.
+ /// Serialize case-insensitive keys before checking, including concurrent creation.
+ /// Return AlreadyPresent only for an unambiguous matching key, repository ID, and number.
+ /// Different-key stable identities, conflicting IDs, and unverified existing rows conflict.
+ /// Never update, delete, or rename existing rows, including their timestamps.
+ fn initialize_indexed_row(
+ &self,
+ row: &GithubPullRequestRow,
+ ) -> impl Future<Output = Result<PullRequestIndexOutcome, Self::Err>> + Send;
}
/// Lists the GitHub pull requests a caller can see.
diff --git a/crates/github_pull_requests/src/domain/service.rs b/crates/github_pull_requests/src/domain/service.rs
index 1138430926..7cc48a59d5 100644
--- a/crates/github_pull_requests/src/domain/service.rs
+++ b/crates/github_pull_requests/src/domain/service.rs
@@ -4,6 +4,7 @@
mod test;
mod changes;
+mod index;
pub use changes::{GithubPullRequestChangesServiceImpl, GithubPullRequestChangesetStore};
@@ -22,12 +23,10 @@ use super::{
EnrichedGithubPullRequest, GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,
GithubPullRequestError, GithubPullRequestFacets, GithubPullRequestRow,
GithubPullRequestSortDirection, GithubPullRequestStatus, GithubPullRequestWrite,
- GithubRepositoryIdentity, StoredGithubPullRequest, UpsertGithubPullRequest,
- UpsertedGithubPullRequest,
+ StoredGithubPullRequest, UpsertGithubPullRequest, UpsertedGithubPullRequest,
},
ports::{
- GithubPullRequestFacetRepository, GithubPullRequestFacetService,
- GithubPullRequestIndexRepository, GithubPullRequestIndexer, GithubPullRequestListing,
+ GithubPullRequestFacetRepository, GithubPullRequestFacetService, GithubPullRequestListing,
GithubPullRequestListingRepository, GithubPullRequestRepository, GithubPullRequestService,
},
};
@@ -275,49 +274,6 @@ impl<F: ForeignEntityService, R: GithubPullRequestRepository> GithubPullRequestS
}
}
-impl<F, R> GithubPullRequestIndexer for GithubPullRequestServiceImpl<F, R>
-where
- F: ForeignEntityService,
- R: GithubPullRequestRepository + GithubPullRequestIndexRepository,
-{
- #[tracing::instrument(err, skip(self, repositories), fields(repositories = repositories.len()))]
- async fn index_repositories(
- &self,
- repositories: &[GithubRepositoryIdentity],
- ) -> Result<u64, GithubPullRequestError> {
- let mut indexed = 0;
- for repository in repositories {
- let Some(repository_id) = i64::try_from(repository.id).ok().filter(|id| *id != 0)
- else {
- continue;
- };
- let pull_requests = GithubPullRequestIndexRepository::latest_pull_request_metadata(
- &self.repo,
- &repository.owner,
- &repository.name,
- )
- .await
- .map_err(repository_error)?;
- for metadata in &pull_requests {
- let Some(mut row) = GithubPullRequestWrite::from_metadata(metadata) else {
- tracing::warn!("pull request metadata has no typed columns");
- continue;
- };
- row.repository_id = Some(repository_id);
- match self.repo.upsert_row(&row).await {
- Ok(()) => indexed += 1,
- Err(error) => tracing::error!(
- error=?error, github_key=%row.github_key,
- "failed to store pull request row"
- ),
- }
- }
- }
-
- Ok(indexed)
- }
-}
-
impl<F, R> GithubPullRequestListing for GithubPullRequestServiceImpl<F, R>
where
F: ForeignEntityService,
diff --git a/crates/github_pull_requests/src/domain/service/index.rs b/crates/github_pull_requests/src/domain/service/index.rs
new file mode 100644
index 0000000000..a1069826db
--- /dev/null
+++ b/crates/github_pull_requests/src/domain/service/index.rs
@@ -0,0 +1,148 @@
+//! Verified source reconstruction; timestamps never establish repository identity.
+
+use std::collections::BTreeMap;
+
+use super::GithubPullRequestServiceImpl;
+use crate::domain::{
+ models::{
+ GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, GithubKey, GithubPullRequestError,
+ GithubPullRequestRow, GithubPullRequestWrite, GithubRepositoryIdentity,
+ PullRequestIndexOutcome, PullRequestIndexRecord, PullRequestIndexSummary, RepositorySlug,
+ },
+ ports::{GithubPullRequestIndexRepository, GithubPullRequestIndexer},
+};
+use foreign_entity::domain::ports::ForeignEntityService;
+
+impl<F, R> GithubPullRequestIndexer for GithubPullRequestServiceImpl<F, R>
+where
+ F: ForeignEntityService,
+ R: GithubPullRequestIndexRepository,
+{
+ #[tracing::instrument(err, skip(self, repositories), fields(repositories = repositories.len()))]
+ async fn index_repositories(
+ &self,
+ repositories: &[GithubRepositoryIdentity],
+ ) -> Result<PullRequestIndexSummary, GithubPullRequestError> {
+ let mut summary = PullRequestIndexSummary::default();
+ for repository in repositories {
+ let valid_slug =
+ RepositorySlug::parse(&format!("{}/{}", repository.owner, repository.name))
+ .is_some_and(|slug| {
+ slug.owner == repository.owner && slug.name == repository.name
+ });
+ if !valid_slug {
+ summary.failures += 1;
+ tracing::warn!(
+ repository_id = repository.id,
+ "invalid installation repository name"
+ );
+ continue;
+ }
+ let Some(repository_id) = i64::try_from(repository.id).ok().filter(|id| *id > 0) else {
+ summary.failures += 1;
+ tracing::warn!(
+ repository_id = repository.id,
+ "invalid installation repository identity"
+ );
+ continue;
+ };
+ let mut records = match self
+ .repo
+ .pull_request_index_records(&repository.owner, &repository.name)
+ .await
+ {
+ Ok(records) => records,
+ Err(error) => {
+ summary.failures += 1;
+ tracing::error!(error=?error, repository_id, "failed to read pull request index sources");
+ continue;
+ }
+ };
+ records.sort_by_key(|record| (record.updated_at, record.id));
+ let mut groups: BTreeMap<i64, GithubPullRequestRow> = BTreeMap::new();
+ for record in records {
+ let write = match verified_write(&record, repository, repository_id) {
+ Ok(write) => write,
+ Err(Rejection::Unverified) => {
+ summary.unverified_records += 1;
+ continue;
+ }
+ Err(rejection) => {
+ match rejection {
+ Rejection::Invalid => summary.invalid_records += 1,
+ Rejection::Conflict => summary.identity_conflicts += 1,
+ Rejection::Unverified => unreachable!(),
+ }
+ tracing::warn!(record_id=%record.id, github_key=%record.github_key, ?rejection, repository_id, "rejected pull request index source");
+ continue;
+ }
+ };
+ let number = write.number;
+ let row = write.merge(groups.remove(&number));
+ groups.insert(number, row);
+ }
+ for row in groups.into_values() {
+ match self.repo.initialize_indexed_row(&row).await {
+ Ok(PullRequestIndexOutcome::Inserted) => summary.inserted += 1,
+ Ok(PullRequestIndexOutcome::AlreadyPresent) => summary.already_present += 1,
+ Ok(PullRequestIndexOutcome::IdentityConflict) => {
+ summary.identity_conflicts += 1;
+ tracing::warn!(github_key=%row.github_key, repository_id, "pull request index identity conflicts with existing row");
+ }
+ Err(error) => {
+ summary.failures += 1;
+ tracing::error!(error=?error, github_key=%row.github_key, repository_id, "failed to initialize pull request index row");
+ }
+ }
+ }
+ }
+ Ok(summary)
+ }
+}
+
+#[derive(Debug)]
+enum Rejection {
+ Unverified,
+ Invalid,
+ Conflict,
+}
+
+fn verified_write(
+ record: &PullRequestIndexRecord,
+ repository: &GithubRepositoryIdentity,
+ repository_id: i64,
+) -> Result<GithubPullRequestWrite, Rejection> {
+ if record.source != GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE {
+ return Err(Rejection::Invalid);
+ }
+ let mut write =
+ GithubPullRequestWrite::from_metadata(&record.metadata).ok_or(Rejection::Invalid)?;
+ // Check the original JSON too: an overflowing ID must not become an omitted ID.
+ let source_id = match record.metadata.get("repositoryId") {
+ None | Some(serde_json::Value::Null) => return Err(Rejection::Unverified),
+ Some(value) => value
+ .as_u64()
+ .and_then(|id| i64::try_from(id).ok())
+ .filter(|id| *id > 0)
+ .ok_or(Rejection::Invalid)?,
+ };
+ if source_id != repository_id || write.repository_id != Some(repository_id) {
+ return Err(Rejection::Conflict);
+ }
+ if write.number <= 0 {
+ return Err(Rejection::Invalid);
+ }
+ let key = GithubKey::new(&repository.owner, &repository.name, write.number as u64).to_string();
+ if !write.owner.eq_ignore_ascii_case(&repository.owner)
+ || !write.repo.eq_ignore_ascii_case(&repository.name)
+ || !write.github_key.eq_ignore_ascii_case(&key)
+ || !record.github_key.eq_ignore_ascii_case(&key)
+ {
+ return Err(Rejection::Invalid);
+ }
+ // Canonical casing is only for new rows; initialization never renames an existing row.
+ write.github_key = key;
+ write.owner = repository.owner.clone();
+ write.repo = repository.name.clone();
+ Ok(write)
+}
diff --git a/crates/github_pull_requests/src/domain/service/test.rs b/crates/github_pull_requests/src/domain/service/test.rs
index f87b3e045d..917afae144 100644
--- a/crates/github_pull_requests/src/domain/service/test.rs
+++ b/crates/github_pull_requests/src/domain/service/test.rs
@@ -1,3 +1,4 @@
+mod index;
mod initialization;
mod refresh;
use std::convert::Infallible;
@@ -26,7 +27,7 @@ use crate::domain::{
GithubPullRequestError, GithubPullRequestFacets, GithubPullRequestLabel,
GithubPullRequestReviewDecision, GithubPullRequestRow, GithubPullRequestSortDirection,
GithubPullRequestStatus, GithubPullRequestWrite, GithubRepositoryIdentity,
- UpsertGithubPullRequest,
+ PullRequestIndexOutcome, PullRequestIndexRecord, UpsertGithubPullRequest,
},
ports::{
GithubPullRequestFacetRepository, GithubPullRequestFacetService,
@@ -171,8 +172,9 @@ impl ForeignEntityService for StubForeignEntityService {
#[derive(Clone, Default)]
struct StubPullRequestRows {
rows: Arc<Mutex<Vec<GithubPullRequestRow>>>,
- /// Stored pull request metadata by repository owner and name.
- stored: Arc<Mutex<Vec<(String, String, serde_json::Value)>>>,
+ stored: Arc<Mutex<Vec<PullRequestIndexRecord>>>,
+ index_read_failures: Arc<Mutex<Vec<String>>>,
+ index_write_failures: Arc<Mutex<Vec<i64>>>,
listing_calls: Arc<Mutex<Vec<ListingCall>>>,
fail_listings: Arc<Mutex<bool>>,
facet_requests: Arc<Mutex<Vec<Vec<SourceId>>>>,
@@ -250,22 +252,63 @@ impl GithubPullRequestRepository for StubPullRequestRows {
}
impl GithubPullRequestIndexRepository for StubPullRequestRows {
- type Err = Infallible;
+ type Err = anyhow::Error;
- async fn latest_pull_request_metadata(
+ async fn pull_request_index_records(
&self,
owner: &str,
name: &str,
- ) -> Result<Vec<serde_json::Value>, Self::Err> {
+ ) -> Result<Vec<PullRequestIndexRecord>, Self::Err> {
+ let prefix = format!("{owner}/{name}/pull/").to_ascii_lowercase();
+ if self.index_read_failures.lock().unwrap().contains(&prefix) {
+ anyhow::bail!("source read failed");
+ }
Ok(self
.stored
.lock()
.unwrap()
.iter()
- .filter(|(stored_owner, stored_name, _)| stored_owner == owner && stored_name == name)
- .map(|(_, _, metadata)| metadata.clone())
+ .filter(|record| record.github_key.to_ascii_lowercase().starts_with(&prefix))
+ .cloned()
.collect())
}
+
+ async fn initialize_indexed_row(
+ &self,
+ row: &GithubPullRequestRow,
+ ) -> Result<PullRequestIndexOutcome, Self::Err> {
+ if self
+ .index_write_failures
+ .lock()
+ .unwrap()
+ .contains(&row.number)
+ {
+ anyhow::bail!("initialization failed");
+ }
+ let mut rows = self.rows.lock().unwrap();
+ let existing: Vec<_> = rows
+ .iter()
+ .filter(|stored| {
+ stored.github_key.eq_ignore_ascii_case(&row.github_key)
+ || (stored.repository_id == row.repository_id && stored.number == row.number)
+ })
+ .collect();
+ if existing.is_empty() {
+ rows.push(row.clone());
+ return Ok(PullRequestIndexOutcome::Inserted);
+ }
+ Ok(
+ if existing.len() == 1
+ && existing[0].github_key.eq_ignore_ascii_case(&row.github_key)
+ && existing[0].repository_id == row.repository_id
+ && existing[0].number == row.number
+ {
+ PullRequestIndexOutcome::AlreadyPresent
+ } else {
+ PullRequestIndexOutcome::IdentityConflict
+ },
+ )
+ }
}
impl GithubPullRequestListingRepository for StubPullRequestRows {
@@ -679,58 +722,6 @@ async fn upsert_after_a_repository_rename_moves_every_record_and_the_row() {
assert_eq!(stored_rows[0].status, Some(GithubPullRequestStatus::Merged));
}
-#[tokio::test]
-async fn index_writes_a_row_for_each_stored_pull_request_of_each_known_repository() {
- let metadata = |key: &str, number: u64, status: &str| {
- serde_json::json!({
- "githubKey": key,
- "owner": "Macro",
- "repo": "App",
- "number": number,
- "url": format!("https://github.com/{key}"),
- "displayName": key,
- "status": status,
- })
- };
- let rows = StubPullRequestRows::default();
- rows.stored.lock().unwrap().extend([
- (
- "macro".to_string(),
- "app".to_string(),
- metadata("Macro/App/pull/7", 7, "closed"),
- ),
- (
- "macro".to_string(),
- "app-web".to_string(),
- metadata("macro/app-web/pull/1", 1, "open"),
- ),
- ]);
- let service = service(&StubForeignEntityService::default(), &rows);
-
- let indexed = service
- .index_repositories(&[
- GithubRepositoryIdentity {
- id: 99,
- owner: "macro".to_string(),
- name: "app".to_string(),
- },
- GithubRepositoryIdentity {
- id: 0,
- owner: "macro".to_string(),
- name: "app-web".to_string(),
- },
- ])
- .await
- .unwrap();
-
- assert_eq!(indexed, 1);
- let stored_rows = rows.rows();
- assert_eq!(stored_rows.len(), 1);
- assert_eq!(stored_rows[0].github_key, "Macro/App/pull/7");
- assert_eq!(stored_rows[0].repository_id, Some(99));
- assert_eq!(stored_rows[0].status, Some(GithubPullRequestStatus::Closed));
-}
-
fn listing_query() -> ForeignEntityListQuery {
Query::Sort(SimpleSortMethod::UpdatedAt, None)
}
@@ -906,47 +897,6 @@ async fn facets_reject_a_receipt_for_anything_but_a_team() {
assert!(rows.facet_requests.lock().unwrap().is_empty());
}
-#[tokio::test]
-async fn index_rebuild_keeps_richer_shared_row_when_latest_record_is_sparse() {
- let rows = StubPullRequestRows::default();
- let rich = serde_json::json!({
- "githubKey": GITHUB_KEY, "owner": "macro", "repo": "app", "number": 7,
- "url": "https://github.com/macro/app/pull/7", "displayName": "PR",
- "draft": true, "assignees": [{"githubUserId":"7"}], "labels": [{"name":"bug"}],
- "requestedReviewerGithubUserIds": ["8"], "participantGithubUserIds": ["7"]
- });
- rows.rows
- .lock()
- .unwrap()
- .push(GithubPullRequestRow::from_metadata(&rich).unwrap());
- rows.stored.lock().unwrap().push((
- "macro".into(),
- "app".into(),
- serde_json::json!({
- "githubKey": GITHUB_KEY, "owner": "macro", "repo": "app", "number": 7,
- "url": "https://github.com/macro/app/pull/7", "displayName": "PR",
- "status": "closed"
- }),
- ));
- let service = service(&StubForeignEntityService::default(), &rows);
- assert_eq!(
- service
- .index_repositories(&[GithubRepositoryIdentity {
- id: 99,
- owner: "macro".into(),
- name: "app".into(),
- }])
- .await
- .unwrap(),
- 1
- );
- let result = rows.rows();
- assert!(result[0].draft);
- assert_eq!(result[0].assignees.len(), 1);
- assert_eq!(result[0].labels.len(), 1);
- assert_eq!(result[0].requested_reviewer_github_user_ids, vec!["8"]);
- assert_eq!(result[0].status, Some(GithubPullRequestStatus::Closed));
-}
fn view_receipt(record: &ForeignEntity) -> EntityAccessReceipt<ViewAccessLevel> {
EntityAccessReceipt::<ViewAccessLevel>::dangerously_assert_authenticated_user(
facet_user(),
diff --git a/crates/github_pull_requests/src/domain/service/test/index.rs b/crates/github_pull_requests/src/domain/service/test/index.rs
new file mode 100644
index 0000000000..8de2cbf8b9
--- /dev/null
+++ b/crates/github_pull_requests/src/domain/service/test/index.rs
@@ -0,0 +1,241 @@
+use super::*;
+
+fn candidate(id: u128, key: &str) -> PullRequestIndexRecord {
+ let parts: Vec<_> = key.split('/').collect();
+ PullRequestIndexRecord {
+ id: Uuid::from_u128(id),
+ source: GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE.into(),
+ github_key: key.into(),
+ updated_at: DateTime::from_timestamp(id as i64, 0).unwrap(),
+ metadata: serde_json::json!({
+ "githubKey": key, "repositoryId": 99, "owner": parts[0], "repo": parts[1],
+ "number": parts[3].parse::<u64>().unwrap(),
+ "url": format!("https://github.com/{key}"), "displayName": "PR", "status": "open"
+ }),
+ }
+}
+
+fn repository() -> GithubRepositoryIdentity {
+ GithubRepositoryIdentity {
+ id: 99,
+ owner: "macro".into(),
+ name: "app".into(),
+ }
+}
+
+async fn index(rows: &StubPullRequestRows) -> crate::domain::models::PullRequestIndexSummary {
+ service(&StubForeignEntityService::default(), rows)
+ .index_repositories(&[repository()])
+ .await
+ .unwrap()
+}
+
+#[tokio::test]
+async fn missing_row_recovers_all_verified_sources_and_case_variants() {
+ let rows = StubPullRequestRows::default();
+ let mut rich = candidate(1, GITHUB_KEY);
+ rich.metadata["draft"] = true.into();
+ rich.metadata["labels"] = serde_json::json!([{"name":"bug"}]);
+ rich.metadata["reviews"] = serde_json::json!([{
+ "id": 1, "reviewerGithubUserId": "42", "state": "approved",
+ "submittedAt": "2026-01-01T00:00:00Z"
+ }]);
+ rich.metadata["base"] = serde_json::json!({"name":"main", "sha":"base-sha"});
+ rich.metadata["head"] = serde_json::json!({"name":"feature", "sha":"head-sha"});
+ let mut sparse = candidate(2, "Macro/App/pull/7");
+ sparse.metadata["status"] = "closed".into();
+ rows.stored.lock().unwrap().extend([sparse, rich]);
+ let summary = index(&rows).await;
+ assert_eq!(summary.inserted, 1);
+ assert!(!summary.has_failures());
+ let result = rows.rows();
+ assert_eq!(result.len(), 1);
+ assert!(result[0].draft);
+ assert_eq!(result[0].labels.len(), 1);
+ assert_eq!(
+ result[0].review_decision,
+ Some(GithubPullRequestReviewDecision::Approved)
+ );
+ assert_eq!(result[0].status, Some(GithubPullRequestStatus::Closed));
+ assert_eq!(
+ result[0].head.as_ref().unwrap().sha.as_deref(),
+ Some("head-sha")
+ );
+}
+
+#[tokio::test]
+async fn explicit_empty_fields_clear_and_refs_replace_as_pairs() {
+ let rows = StubPullRequestRows::default();
+ let mut rich = candidate(1, GITHUB_KEY);
+ rich.metadata["assignees"] = serde_json::json!([{"githubUserId":"42"}]);
+ rich.metadata["labels"] = serde_json::json!([{"name":"bug"}]);
+ rich.metadata["requestedReviewerGithubUserIds"] = serde_json::json!(["42"]);
+ rich.metadata["base"] = serde_json::json!({"name":"main", "sha":"old-base"});
+ rich.metadata["head"] = serde_json::json!({"name":"old-feature", "sha":"old-head"});
+ let mut update = candidate(2, GITHUB_KEY);
+ for field in ["assignees", "labels", "requestedReviewerGithubUserIds"] {
+ update.metadata[field] = serde_json::json!([]);
+ }
+ update.metadata["base"] = serde_json::json!({"name":"new-base"});
+ update.metadata["head"] = serde_json::json!({"sha":"new-head"});
+ rows.stored.lock().unwrap().extend([rich, update]);
+ assert_eq!(index(&rows).await.inserted, 1);
+ let row = &rows.rows()[0];
+ assert!(
+ row.assignees.is_empty()
+ && row.labels.is_empty()
+ && row.requested_reviewer_github_user_ids.is_empty()
+ );
+ assert_eq!(row.base.as_ref().unwrap().name.as_deref(), Some("new-base"));
+ assert_eq!(row.base.as_ref().unwrap().sha, None);
+ assert_eq!(row.head.as_ref().unwrap().name, None);
+ assert_eq!(row.head.as_ref().unwrap().sha.as_deref(), Some("new-head"));
+}
+
+#[tokio::test]
+async fn legacy_and_reused_repository_names_never_supply_identity() {
+ let rows = StubPullRequestRows::default();
+ let mut legacy = candidate(1, GITHUB_KEY);
+ legacy
+ .metadata
+ .as_object_mut()
+ .unwrap()
+ .remove("repositoryId");
+ legacy.metadata["labels"] = serde_json::json!([{"name":"unverified"}]);
+ let mut reused = candidate(2, GITHUB_KEY);
+ reused.metadata["repositoryId"] = 100.into();
+ rows.stored.lock().unwrap().extend([legacy, reused]);
+ let result = index(&rows).await;
+ assert_eq!(result.unverified_records, 1);
+ assert_eq!(result.identity_conflicts, 1);
+ assert!(result.has_failures());
+ assert!(rows.rows().is_empty());
+ rows.stored.lock().unwrap().push(candidate(3, GITHUB_KEY));
+ assert_eq!(index(&rows).await.inserted, 1);
+ assert!(rows.rows()[0].labels.is_empty());
+}
+
+#[tokio::test]
+async fn malformed_keys_sources_and_numeric_ids_are_rejected() {
+ let rows = StubPullRequestRows::default();
+ let mut records = Vec::new();
+ for (field, value) in [
+ ("repositoryId", serde_json::json!(0)),
+ ("repositoryId", serde_json::json!(u64::MAX)),
+ ("number", serde_json::json!(0)),
+ ("number", serde_json::json!(u64::MAX)),
+ ("owner", serde_json::json!("other")),
+ ("repo", serde_json::json!("other")),
+ ("githubKey", serde_json::json!("macro/app/pull/8")),
+ ] {
+ let mut record = candidate(1, GITHUB_KEY);
+ record.metadata[field] = value;
+ records.push(record);
+ }
+ let mut wrong_source = candidate(2, GITHUB_KEY);
+ wrong_source.source = "linear_issue".into();
+ records.push(wrong_source);
+ let mut wrong_key = candidate(3, "macro/app/pull/8");
+ wrong_key.metadata = candidate(3, GITHUB_KEY).metadata;
+ records.push(wrong_key);
+ rows.stored.lock().unwrap().extend(records);
+ let result = index(&rows).await;
+ assert_eq!(result.invalid_records, 9);
+ assert!(rows.rows().is_empty());
+}
+
+#[tokio::test]
+async fn retries_leave_existing_typed_values_untouched() {
+ let rows = StubPullRequestRows::default();
+ rows.stored.lock().unwrap().push(candidate(1, GITHUB_KEY));
+ assert_eq!(index(&rows).await.inserted, 1);
+ let before = rows.rows();
+ rows.stored.lock().unwrap()[0].metadata["status"] = "closed".into();
+ let result = index(&rows).await;
+ assert_eq!(result.inserted, 0);
+ assert_eq!(result.already_present, 1);
+ assert_eq!(rows.rows(), before);
+}
+
+#[tokio::test]
+async fn existing_unverified_or_conflicting_rows_are_not_reassigned() {
+ for id in [None, Some(100)] {
+ let rows = StubPullRequestRows::default();
+ let mut existing =
+ GithubPullRequestRow::from_metadata(&candidate(1, GITHUB_KEY).metadata).unwrap();
+ existing.repository_id = id;
+ rows.rows.lock().unwrap().push(existing.clone());
+ rows.stored.lock().unwrap().push(candidate(2, GITHUB_KEY));
+ assert_eq!(index(&rows).await.identity_conflicts, 1);
+ assert_eq!(rows.rows(), vec![existing]);
+ }
+}
+
+#[tokio::test]
+async fn tied_source_times_use_record_id_not_input_order() {
+ let rows = StubPullRequestRows::default();
+ let mut older = candidate(1, GITHUB_KEY);
+ older.metadata["name"] = "earlier-id".into();
+ let mut newer = candidate(2, GITHUB_KEY);
+ newer.updated_at = older.updated_at;
+ newer.metadata["name"] = "later-id".into();
+ rows.stored.lock().unwrap().extend([newer, older]);
+ assert_eq!(index(&rows).await.inserted, 1);
+ assert_eq!(rows.rows()[0].title.as_deref(), Some("later-id"));
+}
+
+#[tokio::test]
+async fn failures_do_not_stop_independent_records_or_repositories() {
+ let rows = StubPullRequestRows::default();
+ rows.index_write_failures.lock().unwrap().push(7);
+ rows.index_read_failures
+ .lock()
+ .unwrap()
+ .push("macro/broken/pull/".into());
+ rows.stored
+ .lock()
+ .unwrap()
+ .extend([candidate(1, GITHUB_KEY), candidate(2, "macro/app/pull/8")]);
+ let result = service(&StubForeignEntityService::default(), &rows)
+ .index_repositories(&[
+ GithubRepositoryIdentity {
+ id: 99,
+ owner: "macro".into(),
+ name: "broken".into(),
+ },
+ repository(),
+ GithubRepositoryIdentity {
+ id: u64::MAX,
+ ..repository()
+ },
+ GithubRepositoryIdentity {
+ id: 0,
+ ..repository()
+ },
+ ])
+ .await
+ .unwrap();
+ assert_eq!(result.inserted, 1);
+ assert_eq!(result.failures, 4);
+ assert_eq!(rows.rows()[0].number, 8);
+}
+
+#[tokio::test]
+async fn invalid_installation_repository_names_are_rejected_before_scanning() {
+ let rows = StubPullRequestRows::default();
+ let invalid = [("", "app"), ("macro/other", "app"), ("macro", "app/pull/7")]
+ .into_iter()
+ .map(|(owner, name)| GithubRepositoryIdentity {
+ id: 99,
+ owner: owner.into(),
+ name: name.into(),
+ })
+ .collect::<Vec<_>>();
+ let result = service(&StubForeignEntityService::default(), &rows)
+ .index_repositories(&invalid)
+ .await
+ .unwrap();
+ assert_eq!(result.failures, 3);
+ assert_eq!(result.inserted, 0);
+ assert!(rows.rows().is_empty());
+}
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs
index f4816ea657..873b8bc873 100644
--- a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo.rs
@@ -54,7 +54,7 @@ impl GithubPullRequestRepository for PgGithubPullRequestRepo {
let mut tx = self.pool.begin().await?;
// Lock the key before reading, including when no row exists yet.
sqlx::query!(
- "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",
+ "SELECT pg_advisory_xact_lock(hashtextextended(lower($1), 0))",
update.github_key
)
.execute(&mut *tx)
@@ -65,11 +65,39 @@ impl GithubPullRequestRepository for PgGithubPullRequestRepo {
requested_reviewer_github_user_ids, participant_github_user_ids,
github_updated_at, assignees, labels, reviews,
base_ref, base_sha, head_ref, head_sha
- FROM github_pull_request WHERE github_key = $1 FOR UPDATE"#,
+ FROM github_pull_request
+ WHERE github_key = $1
+ OR (lower(github_key) = lower($1) AND repository_id = $2 AND number = $3)
+ ORDER BY github_key
+ FOR UPDATE"#,
update.github_key,
+ update.repository_id,
+ update.number,
)
- .fetch_optional(&mut *tx)
+ .fetch_all(&mut *tx)
.await?;
+ // Index initialization can win after follow_rename observed no row. A case-only
+ // fallback retains the stored key; a real rename still belongs to follow_rename.
+ if stored.len() > 1
+ || stored.iter().any(|row| {
+ row.number != update.number
+ || matches!(
+ (row.repository_id, update.repository_id),
+ (Some(stored_id), Some(incoming_id)) if stored_id != incoming_id
+ )
+ })
+ {
+ return Err(sqlx::Error::Protocol(
+ "pull request typed identity conflict".into(),
+ ));
+ }
+ if stored.is_empty() && sqlx::query_scalar!(
+ "SELECT github_key FROM github_pull_request WHERE lower(github_key) = lower($1) LIMIT 1 FOR UPDATE",
+ update.github_key,
+ ).fetch_optional(&mut *tx).await?.is_some() {
+ return Err(sqlx::Error::Protocol("pull request case-variant identity conflict".into()));
+ }
+ let stored = stored.into_iter().next();
let decode = |value: serde_json::Value| {
serde_json::from_value(value).map_err(|error| sqlx::Error::Decode(error.into()))
};
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs
index ce6119bf2b..a496dfcd08 100644
--- a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index.rs
@@ -1,36 +1,192 @@
-//! The stored pull request records the indexer writes rows from.
+//! Stored source scans and insert-only initialization of verified typed rows.
#[cfg(test)]
mod test;
+use sqlx::{Postgres, Transaction};
+
use super::PgGithubPullRequestRepo;
use crate::domain::{
- models::GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, ports::GithubPullRequestIndexRepository,
+ models::{
+ GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE, GithubPullRequestRow, PullRequestIndexOutcome,
+ PullRequestIndexRecord,
+ },
+ ports::GithubPullRequestIndexRepository,
};
-/// The records belong to the foreign_entity crate; the indexer only reads them.
+const SOURCE_PAGE_SIZE: i64 = 512;
+
+/// Foreign-entity records are only read; initialization writes this crate's typed table.
impl GithubPullRequestIndexRepository for PgGithubPullRequestRepo {
type Err = sqlx::Error;
#[tracing::instrument(err, skip(self))]
- async fn latest_pull_request_metadata(
+ async fn pull_request_index_records(
&self,
owner: &str,
name: &str,
- ) -> Result<Vec<serde_json::Value>, Self::Err> {
+ ) -> Result<Vec<PullRequestIndexRecord>, Self::Err> {
let key_prefix = format!("{owner}/{name}/pull/");
- sqlx::query_scalar!(
+ let mut after: Option<uuid::Uuid> = None;
+ let mut records = Vec::new();
+ loop {
+ let page = sqlx::query_as!(
+ PullRequestIndexRecord,
+ r#"
+ SELECT id, foreign_entity_source AS source, foreign_entity_id AS github_key,
+ updated_at, metadata
+ FROM foreign_entity
+ WHERE foreign_entity_source = $1::text
+ AND left(lower(foreign_entity_id), length($2::text)) = lower($2::text)
+ AND ($3::uuid IS NULL OR id > $3)
+ ORDER BY id ASC
+ LIMIT $4
+ "#,
+ GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,
+ key_prefix,
+ after,
+ SOURCE_PAGE_SIZE,
+ )
+ .fetch_all(&self.pool)
+ .await?;
+ let complete = page.len() < SOURCE_PAGE_SIZE as usize;
+ after = page.last().map(|record| record.id);
+ records.extend(page);
+ if complete {
+ return Ok(records);
+ }
+ }
+ }
+
+ #[tracing::instrument(err, skip(self, row), fields(github_key = %row.github_key))]
+ async fn initialize_indexed_row(
+ &self,
+ row: &GithubPullRequestRow,
+ ) -> Result<PullRequestIndexOutcome, Self::Err> {
+ let repository_id = row.repository_id.filter(|id| *id > 0).ok_or_else(|| {
+ sqlx::Error::Protocol(
+ "index initialization requires verified repository identity".into(),
+ )
+ })?;
+ if row.number <= 0 {
+ return Err(sqlx::Error::Protocol(
+ "index initialization requires a positive PR number".into(),
+ ));
+ }
+ let mut tx = self.pool.begin().await?;
+ // Same case-insensitive lock as live upserts, before reading even an absent key.
+ sqlx::query!(
+ "SELECT pg_advisory_xact_lock(hashtextextended(lower($1), 0))",
+ row.github_key
+ )
+ .execute(&mut *tx)
+ .await?;
+ let identity_lock = format!(
+ "github_pull_request.identity:{repository_id}/{}",
+ row.number
+ );
+ sqlx::query!(
+ "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",
+ identity_lock
+ )
+ .execute(&mut *tx)
+ .await?;
+ if let Some(outcome) = existing_index_outcome(&mut tx, row, repository_id).await? {
+ tx.commit().await?;
+ return Ok(outcome);
+ }
+ let json = |value: serde_json::Result<serde_json::Value>| {
+ value.map_err(|error| sqlx::Error::Encode(error.into()))
+ };
+ let assignees = json(serde_json::to_value(&row.assignees))?;
+ let labels = json(serde_json::to_value(&row.labels))?;
+ let reviews = json(serde_json::to_value(&row.reviews))?;
+ let inserted = sqlx::query!(
r#"
- SELECT DISTINCT ON (fe.foreign_entity_id) fe.metadata AS "metadata!: serde_json::Value"
- FROM foreign_entity fe
- WHERE fe.foreign_entity_source = $1::text
- AND left(lower(fe.foreign_entity_id), length($2::text)) = lower($2::text)
- ORDER BY fe.foreign_entity_id, fe.updated_at DESC, fe.id DESC
+ INSERT INTO github_pull_request (
+ github_key, repository_id, number, owner, repo, title, status, draft,
+ author_github_user_id, author_login, requested_reviewer_github_user_ids,
+ participant_github_user_ids, github_updated_at, assignees, labels, reviews,
+ review_decision, base_ref, base_sha, head_ref, head_sha
+ ) VALUES (
+ $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
+ $14::jsonb, $15::jsonb, $16::jsonb, $17, $18, $19, $20, $21
+ )
+ ON CONFLICT DO NOTHING
"#,
- GITHUB_PULL_REQUEST_FOREIGN_ENTITY_SOURCE,
- key_prefix,
+ row.github_key,
+ repository_id,
+ row.number,
+ row.owner,
+ row.repo,
+ row.title.as_deref(),
+ row.status.map(|status| status.as_str()),
+ row.draft,
+ row.author_github_user_id.as_deref(),
+ row.author_login.as_deref(),
+ &row.requested_reviewer_github_user_ids,
+ &row.participant_github_user_ids,
+ row.github_updated_at,
+ assignees,
+ labels,
+ reviews,
+ row.review_decision.map(|decision| decision.as_str()),
+ row.base.as_ref().and_then(|base| base.name.as_deref()),
+ row.base.as_ref().and_then(|base| base.sha.as_deref()),
+ row.head.as_ref().and_then(|head| head.name.as_deref()),
+ row.head.as_ref().and_then(|head| head.sha.as_deref()),
)
- .fetch_all(&self.pool)
- .await
+ .execute(&mut *tx)
+ .await?
+ .rows_affected()
+ == 1;
+ let outcome = if inserted {
+ PullRequestIndexOutcome::Inserted
+ } else {
+ // A non-index writer can insert another key for this identity between reads.
+ existing_index_outcome(&mut tx, row, repository_id)
+ .await?
+ .unwrap_or(PullRequestIndexOutcome::IdentityConflict)
+ };
+ tx.commit().await?;
+ Ok(outcome)
+ }
+}
+
+async fn existing_index_outcome(
+ tx: &mut Transaction<'_, Postgres>,
+ row: &GithubPullRequestRow,
+ repository_id: i64,
+) -> Result<Option<PullRequestIndexOutcome>, sqlx::Error> {
+ let existing = sqlx::query!(
+ r#"
+ SELECT github_key, repository_id, number, owner, repo
+ FROM github_pull_request
+ WHERE lower(github_key) = lower($1)
+ OR (repository_id = $2 AND number = $3)
+ ORDER BY github_key
+ FOR UPDATE
+ "#,
+ row.github_key,
+ repository_id,
+ row.number,
+ )
+ .fetch_all(&mut **tx)
+ .await?;
+ if existing.is_empty() {
+ return Ok(None);
}
+ let matches = existing.len() == 1
+ && existing.iter().all(|stored| {
+ stored.github_key.eq_ignore_ascii_case(&row.github_key)
+ && stored.repository_id == Some(repository_id)
+ && stored.number == row.number
+ && stored.owner.eq_ignore_ascii_case(&row.owner)
+ && stored.repo.eq_ignore_ascii_case(&row.repo)
+ });
+ Ok(Some(if matches {
+ PullRequestIndexOutcome::AlreadyPresent
+ } else {
+ PullRequestIndexOutcome::IdentityConflict
+ }))
}
diff --git a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs
index 4d78918985..7ff52744fb 100644
--- a/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs
+++ b/crates/github_pull_requests/src/outbound/pg_github_pull_request_repo/index/test.rs
@@ -1,129 +1,336 @@
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use serde_json::{Value, json};
use sqlx::PgPool;
+use uuid::Uuid;
-use super::super::PgGithubPullRequestRepo;
-use crate::domain::ports::GithubPullRequestIndexRepository;
+use super::{super::PgGithubPullRequestRepo, SOURCE_PAGE_SIZE};
+use crate::domain::{
+ models::{GithubPullRequestRow, GithubPullRequestWrite, PullRequestIndexOutcome},
+ ports::{GithubPullRequestIndexRepository, GithubPullRequestRepository},
+};
-async fn insert_record(
- pool: &PgPool,
- key: &str,
- source: &str,
- stored_for_id: &str,
- metadata: Value,
- minutes_ago: i32,
-) {
- sqlx::query(
- r#"
- INSERT INTO foreign_entity (
- id, foreign_entity_id, foreign_entity_source, metadata, stored_for_id,
- stored_for_auth_entity, updated_at
- )
- VALUES (gen_random_uuid(), $1, $2, $3, $4, 'user', NOW() - make_interval(mins => $5::int))
- "#,
+async fn insert_record(pool: &PgPool, key: &str, source: &str, metadata: Value) {
+ sqlx::query!(
+ r#"INSERT INTO foreign_entity (id, foreign_entity_id, foreign_entity_source, metadata,
+ stored_for_id, stored_for_auth_entity)
+ VALUES ($1, $2, $3, $4, 'macro|[email protected]', 'user')"#,
+ Uuid::now_v7(),
+ key,
+ source,
+ metadata,
)
- .bind(key)
- .bind(source)
- .bind(metadata)
- .bind(stored_for_id)
- .bind(minutes_ago)
.execute(pool)
.await
- .expect("record should be inserted");
+ .unwrap();
}
-fn sorted(mut metadata: Vec<Value>) -> Vec<Value> {
- metadata.sort_by_key(|value| value.to_string());
- metadata
+fn row(key: &str, repository_id: Option<i64>) -> GithubPullRequestRow {
+ let parts: Vec<_> = key.split('/').collect();
+ GithubPullRequestRow::from_metadata(&json!({
+ "githubKey": key, "owner": parts[0], "repo": parts[1],
+ "number": parts[3].parse::<u64>().unwrap(), "repositoryId": repository_id,
+ "url": format!("https://github.com/{key}"), "displayName": "PR",
+ "name": "seed title", "status": "open", "draft": true,
+ "labels": [{"name":"bug"}], "base": {"name":"main", "sha":"base"},
+ "head": {"name":"feature", "sha":"head"}
+ }))
+ .unwrap()
}
-#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn latest_metadata_covers_each_pull_request_in_the_repository_once(pool: PgPool) {
- let user = "macro|[email protected]";
- let team = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
- insert_record(
- &pool,
- "Macro-Inc/App/pull/1",
- "github_pull_request",
- user,
- json!({ "n": 1 }),
- 0,
- )
- .await;
- insert_record(
- &pool,
- "macro-inc/app/pull/2",
- "github_pull_request",
- user,
- json!({ "n": "older" }),
- 10,
- )
- .await;
- insert_record(
- &pool,
- "macro-inc/app/pull/2",
- "github_pull_request",
- team,
- json!({ "n": "newer" }),
- 1,
+async fn snapshot(pool: &PgPool) -> Vec<Value> {
+ sqlx::query_scalar!(
+ r#"SELECT to_jsonb(pr) AS "snapshot!: serde_json::Value"
+ FROM github_pull_request pr ORDER BY github_key"#
)
- .await;
- insert_record(
- &pool,
- "macro-inc/app-web/pull/3",
- "github_pull_request",
- user,
- json!({ "n": 3 }),
- 0,
- )
- .await;
- insert_record(
- &pool,
- "macro-inc/app/pull/4",
- "linear_issue",
- user,
- json!({ "n": 4 }),
- 0,
- )
- .await;
+ .fetch_all(pool)
+ .await
+ .unwrap()
+}
- let metadata = PgGithubPullRequestRepo::new(pool)
- .latest_pull_request_metadata("macro-inc", "app")
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn candidates_retain_older_cross_source_and_case_variant_metadata(pool: PgPool) {
+ for (key, source, metadata) in [
+ (
+ "Macro-Inc/App/pull/1",
+ "github_pull_request",
+ json!({"n":1}),
+ ),
+ (
+ "macro-inc/app/pull/2",
+ "github_pull_request",
+ json!({"n":"rich"}),
+ ),
+ (
+ "MACRO-INC/APP/pull/2",
+ "github_pull_request",
+ json!({"n":"sparse"}),
+ ),
+ (
+ "macro-inc/app/pull/2",
+ "github_pull_request",
+ json!({"n":"another-source"}),
+ ),
+ (
+ "macro-inc/app-web/pull/3",
+ "github_pull_request",
+ json!({"n":3}),
+ ),
+ ("macro-inc/app/pull/4", "linear_issue", json!({"n":4})),
+ ] {
+ insert_record(&pool, key, source, metadata).await;
+ }
+ let records = PgGithubPullRequestRepo::new(pool)
+ .pull_request_index_records("macro-inc", "app")
.await
- .expect("records should load");
-
- assert_eq!(
- sorted(metadata),
- sorted(vec![json!({ "n": 1 }), json!({ "n": "newer" })])
+ .unwrap();
+ assert_eq!(records.len(), 4);
+ assert!(records.windows(2).all(|pair| pair[0].id < pair[1].id));
+ assert!(
+ records
+ .iter()
+ .all(|record| record.source == "github_pull_request")
);
+ let mut metadata: Vec<_> = records
+ .into_iter()
+ .map(|record| record.metadata.to_string())
+ .collect();
+ metadata.sort();
+ let mut expected: Vec<_> = [
+ json!({"n":1}),
+ json!({"n":"rich"}),
+ json!({"n":"sparse"}),
+ json!({"n":"another-source"}),
+ ]
+ .into_iter()
+ .map(|value| value.to_string())
+ .collect();
+ expected.sort();
+ assert_eq!(metadata, expected);
}
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn repository_names_match_literally(pool: PgPool) {
- let user = "macro|[email protected]";
- insert_record(
- &pool,
+async fn repository_boundaries_and_metacharacters_match_literally(pool: PgPool) {
+ for key in [
"macro-inc/appXrepo/pull/1",
- "github_pull_request",
- user,
- json!({ "n": 1 }),
- 0,
- )
- .await;
- insert_record(
- &pool,
"macro-inc/app_repo/pull/2",
- "github_pull_request",
- user,
- json!({ "n": 2 }),
- 0,
+ "macro-inc/app_repo-extra/pull/3",
+ ] {
+ insert_record(&pool, key, "github_pull_request", json!({})).await;
+ }
+ let records = PgGithubPullRequestRepo::new(pool)
+ .pull_request_index_records("macro-inc", "app_repo")
+ .await
+ .unwrap();
+ assert_eq!(records.len(), 1);
+ assert_eq!(records[0].github_key, "macro-inc/app_repo/pull/2");
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn candidate_scan_reads_past_a_sql_page_without_duplicates(pool: PgPool) {
+ let ids: Vec<_> = (0..SOURCE_PAGE_SIZE + 3).map(|_| Uuid::now_v7()).collect();
+ sqlx::query!(
+ r#"INSERT INTO foreign_entity (id, foreign_entity_id, foreign_entity_source, metadata,
+ stored_for_id, stored_for_auth_entity)
+ SELECT id, 'macro/app/pull/7', 'github_pull_request', '{}'::jsonb, 'user', 'user'
+ FROM unnest($1::uuid[]) AS id"#,
+ &ids,
)
- .await;
+ .execute(&pool)
+ .await
+ .unwrap();
+ let records = PgGithubPullRequestRepo::new(pool)
+ .pull_request_index_records("macro", "app")
+ .await
+ .unwrap();
+ assert_eq!(records.len(), ids.len());
+ assert!(records.windows(2).all(|pair| pair[0].id < pair[1].id));
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn initialization_and_retries_never_rewrite_existing_columns_or_timestamps(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ let original = row("Macro/App/pull/7", Some(99));
+ assert_eq!(
+ repo.initialize_indexed_row(&original).await.unwrap(),
+ PullRequestIndexOutcome::Inserted
+ );
+ let before = snapshot(&pool).await;
+ let mut stale = row("macro/app/pull/7", Some(99));
+ stale.title = Some("stale title".into());
+ stale.labels.clear();
+ assert_eq!(
+ repo.initialize_indexed_row(&stale).await.unwrap(),
+ PullRequestIndexOutcome::AlreadyPresent
+ );
+ assert_eq!(snapshot(&pool).await, before);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn conflicting_keys_unverified_rows_and_real_renames_remain_untouched(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ let original = row("Macro/App/pull/7", Some(99));
+ repo.initialize_indexed_row(&original).await.unwrap();
+ let before = snapshot(&pool).await;
+ for candidate in [
+ row("macro/app/pull/7", Some(100)),
+ row("macro/renamed/pull/7", Some(99)),
+ ] {
+ assert_eq!(
+ repo.initialize_indexed_row(&candidate).await.unwrap(),
+ PullRequestIndexOutcome::IdentityConflict
+ );
+ assert_eq!(snapshot(&pool).await, before);
+ }
+ let legacy = row("macro/app/pull/8", None);
+ repo.upsert_row(&GithubPullRequestWrite::from(&legacy))
+ .await
+ .unwrap();
+ let before = snapshot(&pool).await;
+ assert_eq!(
+ repo.initialize_indexed_row(&row("macro/app/pull/8", Some(99)))
+ .await
+ .unwrap(),
+ PullRequestIndexOutcome::IdentityConflict
+ );
+ assert_eq!(snapshot(&pool).await, before);
+ assert!(
+ repo.initialize_indexed_row(&row("macro/app/pull/9", None))
+ .await
+ .is_err()
+ );
+}
- let metadata = PgGithubPullRequestRepo::new(pool)
- .latest_pull_request_metadata("macro-inc", "app_repo")
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn initialization_winning_after_empty_rename_lookup_allows_case_only_live_update(
+ pool: PgPool,
+) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ assert_eq!(repo.github_key_for(99, 7).await.unwrap(), None);
+ repo.initialize_indexed_row(&row("Macro/App/pull/7", Some(99)))
.await
- .expect("records should load");
+ .unwrap();
+ let mut update = GithubPullRequestWrite::from_metadata(&json!({
+ "githubKey":"macro/app/pull/7", "owner":"macro", "repo":"app", "number":7,
+ "repositoryId":99, "url":"https://github.com/macro/app/pull/7", "displayName":"PR",
+ "status":"closed", "name":"live title"
+ }))
+ .unwrap();
+ repo.upsert_row(&update).await.unwrap();
+ let stored = repo
+ .pull_request_row("Macro/App/pull/7")
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(stored.title.as_deref(), Some("live title"));
+ assert_eq!(stored.github_key, "Macro/App/pull/7");
+ assert_eq!(stored.labels.len(), 1);
+ assert_eq!(snapshot(&pool).await.len(), 1);
+ let before = snapshot(&pool).await;
+ update.repository_id = Some(100);
+ assert!(repo.upsert_row(&update).await.is_err());
+ update.github_key = "Macro/App/pull/7".into();
+ assert!(repo.upsert_row(&update).await.is_err());
+ update.repository_id = Some(99);
+ update.github_key = "macro/renamed/pull/7".into();
+ update.repo = "renamed".into();
+ assert!(repo.upsert_row(&update).await.is_err());
+ assert_eq!(snapshot(&pool).await, before);
+}
- assert_eq!(metadata, vec![json!({ "n": 2 })]);
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn live_first_initialization_preserves_live_values_and_timestamps(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ let mut live = row("macro/app/pull/7", Some(99));
+ live.title = Some("live title".into());
+ repo.upsert_row(&GithubPullRequestWrite::from(&live))
+ .await
+ .unwrap();
+ let before = snapshot(&pool).await;
+ assert_eq!(
+ repo.initialize_indexed_row(&row("Macro/App/pull/7", Some(99)))
+ .await
+ .unwrap(),
+ PullRequestIndexOutcome::AlreadyPresent
+ );
+ assert_eq!(snapshot(&pool).await, before);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn normalized_lock_serializes_initialization_and_live_updates(pool: PgPool) {
+ use std::time::Duration;
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ let mut blocker = pool.begin().await.unwrap();
+ sqlx::query!(
+ "SELECT pg_advisory_xact_lock(hashtextextended(lower($1), 0))",
+ "macro/app/pull/7"
+ )
+ .execute(&mut *blocker)
+ .await
+ .unwrap();
+ let initialize_repo = repo.clone();
+ let mut initialize = tokio::spawn(async move {
+ initialize_repo
+ .initialize_indexed_row(&row("Macro/App/pull/7", Some(99)))
+ .await
+ });
+ assert!(
+ tokio::time::timeout(Duration::from_millis(100), &mut initialize)
+ .await
+ .is_err()
+ );
+ let mut live_row = row("macro/app/pull/7", Some(99));
+ live_row.title = Some("live title".into());
+ let update_repo = repo.clone();
+ let mut update = tokio::spawn(async move {
+ update_repo
+ .upsert_row(&GithubPullRequestWrite::from(&live_row))
+ .await
+ });
+ assert!(
+ tokio::time::timeout(Duration::from_millis(100), &mut update)
+ .await
+ .is_err()
+ );
+ blocker.commit().await.unwrap();
+ initialize.await.unwrap().unwrap();
+ update.await.unwrap().unwrap();
+ assert_eq!(snapshot(&pool).await.len(), 1);
+ let key = repo.github_key_for(99, 7).await.unwrap().unwrap();
+ assert_eq!(
+ repo.pull_request_row(&key)
+ .await
+ .unwrap()
+ .unwrap()
+ .title
+ .as_deref(),
+ Some("live title")
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn simultaneous_case_variant_initializations_insert_once(pool: PgPool) {
+ let repo = PgGithubPullRequestRepo::new(pool.clone());
+ let other = repo.clone();
+ let first = row("Macro/App/pull/7", Some(99));
+ let second = row("macro/app/pull/7", Some(99));
+ let (left, right) = tokio::join!(
+ repo.initialize_indexed_row(&first),
+ other.initialize_indexed_row(&second)
+ );
+ let outcomes = [left.unwrap(), right.unwrap()];
+ assert_eq!(
+ outcomes
+ .iter()
+ .filter(|outcome| **outcome == PullRequestIndexOutcome::Inserted)
+ .count(),
+ 1
+ );
+ assert_eq!(
+ outcomes
+ .iter()
+ .filter(|outcome| **outcome == PullRequestIndexOutcome::AlreadyPresent)
+ .count(),
+ 1
+ );
+ assert_eq!(snapshot(&pool).await.len(), 1);
}