Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

9.5 KB
Raw Blame
//! CLI for markdown document content lifecycle backfill.

use std::time::Duration;

use anyhow::Context as _;
use clap::Parser;
use documents::domain::markdown_backfill::{
    MarkdownBackfillOptions, MarkdownBackfillReport, MarkdownBackfillService,
    MarkdownBackfillStats, MarkdownObjectReadError, MarkdownObjectReader,
};
use documents::domain::models::DocumentError;
use documents::domain::ports::markdown::MarkdownInitializationPort;
use documents::outbound::markdown_init::LexicalSyncMarkdownInitializer;
use documents::outbound::pg_document_repo::PgMarkdownBackfillRepo;
use documents::outbound::s3_markdown_source::S3MarkdownObjectReader;
use lexical_client::LexicalClient;
use macro_env_var::env_vars;
use macro_service_urls::{LexicalServiceUrl, SyncServiceUrl};
use sqlx::postgres::PgPoolOptions;
use sync_service_client::SyncServiceClient;

env_vars! {
    struct DatabaseUrl;
    struct SyncServiceAuthKey;
    struct InternalApiSecretKey;
    struct DocumentStorageBucket;
}

/// Backfill markdown document content lifecycle from sync-service state.
#[derive(Clone, Debug, Parser)]
#[command(version, about)]
struct Args {
    /// Persist updates. Omit for dry-run.
    #[arg(long)]
    apply: bool,

    /// Initialize sync-service for missing markdown documents from S3 source bytes.
    #[arg(long)]
    initialize_missing: bool,

    /// Candidate rows fetched per DB batch.
    #[arg(long, default_value_t = 100)]
    batch_size: i64,

    /// Concurrent per-document sync/S3/init operations per batch.
    #[arg(long, default_value_t = 10)]
    concurrency: usize,

    /// Retries after the first sync-service exists attempt.
    #[arg(long, default_value_t = 2)]
    exists_retries: usize,

    /// Per-attempt sync-service exists timeout in seconds.
    #[arg(long, default_value_t = 10)]
    exists_timeout_secs: u64,

    /// Optional maximum candidate rows to scan.
    #[arg(long)]
    limit: Option<usize>,

    /// Optional exclusive document id cursor.
    #[arg(long)]
    start_after: Option<String>,
}

impl Args {
    fn options(&self) -> anyhow::Result<MarkdownBackfillOptions> {
        anyhow::ensure!(self.batch_size > 0, "--batch-size must be positive");
        anyhow::ensure!(self.concurrency > 0, "--concurrency must be positive");
        anyhow::ensure!(
            self.exists_timeout_secs > 0,
            "--exists-timeout-secs must be positive"
        );

        Ok(MarkdownBackfillOptions {
            apply: self.apply,
            batch_size: self.batch_size,
            concurrency: self.concurrency,
            exists_retries: self.exists_retries,
            exists_timeout: Duration::from_secs(self.exists_timeout_secs),
            limit: self.limit,
            start_after: self.start_after.clone(),
            initialize_missing: self.initialize_missing,
        })
    }
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    macro_entrypoint::MacroEntrypoint::default().init();

    let args = Args::parse();
    let options = args.options()?;
    if !options.apply {
        tracing::warn!("dry-run mode: pass --apply to update Document.contentLocation");
    }

    let database_url = DatabaseUrl::new().context("DATABASE_URL must be set")?;
    let sync_service_auth_key = SyncServiceAuthKey::new()
        .context("SYNC_SERVICE_AUTH_KEY must be set")?
        .to_string();
    let sync_service_url = SyncServiceUrl::new()?.to_string();

    tracing::info!(
        apply = options.apply,
        initialize_missing = options.initialize_missing,
        batch_size = options.batch_size,
        concurrency = options.concurrency,
        exists_retries = options.exists_retries,
        exists_timeout_secs = options.exists_timeout.as_secs(),
        limit = ?options.limit,
        start_after = ?options.start_after,
        sync_service_url = %sync_service_url,
        "starting markdown content-location backfill"
    );

    let db = PgPoolOptions::new()
        .max_connections(5)
        .connect(database_url.as_ref())
        .await
        .context("failed to connect to postgres")?;

    let repo = PgMarkdownBackfillRepo::new(db);
    let sync_probe =
        SyncServiceClient::new(sync_service_auth_key.clone(), sync_service_url.clone());
    let object_reader = build_object_reader(options.initialize_missing).await?;
    let markdown_initializer = build_markdown_initializer(
        options.apply && options.initialize_missing,
        sync_service_auth_key,
        sync_service_url,
    )?;

    let service =
        MarkdownBackfillService::new(repo, sync_probe, object_reader, markdown_initializer);
    let report = service.run(options.clone()).await?;

    tracing::info!(
        scanned = report.stats.scanned,
        sync_exists = report.stats.sync_exists,
        sync_missing = report.stats.sync_missing,
        would_update = report.stats.would_update,
        updated = report.stats.updated,
        would_initialize = report.stats.would_initialize,
        initialized = report.stats.initialized,
        object_missing = report.stats.object_missing,
        object_read_errors = report.stats.object_read_errors,
        invalid_utf8 = report.stats.invalid_utf8,
        initialize_errors = report.stats.initialize_errors,
        missing_document_instance = report.stats.missing_document_instance,
        sync_errors = report.stats.sync_errors,
        last_id = ?report.last_id,
        apply = options.apply,
        "markdown content-location backfill complete"
    );
    print_summary(&report, options.apply);

    Ok(())
}

fn build_markdown_initializer(
    enabled: bool,
    sync_service_auth_key: String,
    sync_service_url: String,
) -> anyhow::Result<OptionalMarkdownInitializer> {
    if !enabled {
        return Ok(OptionalMarkdownInitializer::Disabled);
    }

    let internal_api_secret_key = InternalApiSecretKey::new()
        .context("INTERNAL_API_SECRET_KEY must be set when --initialize-missing is used")?
        .to_string();
    let lexical_service_url = LexicalServiceUrl::new()?.to_string();

    Ok(OptionalMarkdownInitializer::Enabled(
        LexicalSyncMarkdownInitializer::new(
            LexicalClient::new(internal_api_secret_key, lexical_service_url),
            SyncServiceClient::new(sync_service_auth_key, sync_service_url),
        ),
    ))
}

async fn build_object_reader(enabled: bool) -> anyhow::Result<OptionalMarkdownObjectReader> {
    if !enabled {
        return Ok(OptionalMarkdownObjectReader::Disabled);
    }

    let document_storage_bucket = DocumentStorageBucket::new()
        .context("DOCUMENT_STORAGE_BUCKET must be set when --initialize-missing is used")?
        .to_string();

    Ok(OptionalMarkdownObjectReader::Enabled(
        S3MarkdownObjectReader::new(document_storage_bucket, macro_aws_config::s3_client().await),
    ))
}

#[derive(Clone)]
enum OptionalMarkdownObjectReader {
    Disabled,
    Enabled(S3MarkdownObjectReader),
}

impl MarkdownObjectReader for OptionalMarkdownObjectReader {
    async fn read_markdown(
        &self,
        candidate: &documents::domain::markdown_backfill::MarkdownBackfillCandidate,
    ) -> Result<String, MarkdownObjectReadError> {
        match self {
            Self::Disabled => Err(MarkdownObjectReadError::Read {
                key: "<disabled>".to_string(),
                error: "markdown object reader disabled".to_string(),
            }),
            Self::Enabled(reader) => reader.read_markdown(candidate).await,
        }
    }
}

#[derive(Clone)]
enum OptionalMarkdownInitializer {
    Disabled,
    Enabled(LexicalSyncMarkdownInitializer),
}

impl MarkdownInitializationPort for OptionalMarkdownInitializer {
    async fn initialize_existing_markdown(
        &self,
        document_id: &str,
        markdown: &str,
    ) -> Result<Vec<u8>, DocumentError> {
        match self {
            Self::Disabled => Err(DocumentError::Internal(anyhow::anyhow!(
                "markdown initializer disabled"
            ))),
            Self::Enabled(initializer) => {
                initializer
                    .initialize_existing_markdown(document_id, markdown)
                    .await
            }
        }
    }

    async fn initialize_existing_notion(
        &self,
        _document_id: &str,
        _notion: &documents::domain::notion_markdown::NotionMarkdown,
    ) -> Result<documents::domain::ports::markdown::MarkdownSnapshot, DocumentError> {
        unimplemented!("the markdown backfill never imports Notion pages")
    }
}

fn print_summary(report: &MarkdownBackfillReport, apply: bool) {
    let MarkdownBackfillStats {
        scanned,
        sync_exists,
        sync_missing,
        would_update,
        updated,
        would_initialize,
        initialized,
        object_missing,
        object_read_errors,
        invalid_utf8,
        initialize_errors,
        missing_document_instance,
        sync_errors,
    } = report.stats;

    println!("\nMarkdown backfill summary");
    println!("  apply: {apply}");
    println!("  scanned: {scanned}");
    println!("  sync_exists: {sync_exists}");
    println!("  sync_missing: {sync_missing}");
    println!("  would_update: {would_update}");
    println!("  updated: {updated}");
    println!("  would_initialize: {would_initialize}");
    println!("  initialized: {initialized}");
    println!("  object_missing: {object_missing}");
    println!("  object_read_errors: {object_read_errors}");
    println!("  invalid_utf8: {invalid_utf8}");
    println!("  initialize_errors: {initialize_errors}");
    println!("  missing_document_instance: {missing_document_instance}");
    println!("  sync_errors: {sync_errors}");
    println!("  last_id: {:?}", report.last_id);
}