:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::path::PathBuf;
use clap::Subcommand;
#[derive(Debug, Subcommand)]
pub enum LogsCommand {
/// Import historical logs from another log store into PostHog
Import {
#[command(subcommand)]
cmd: ImportSource,
},
}
#[derive(Debug, Subcommand)]
pub enum ImportSource {
/// Read from Grafana Loki. Credentials come from LOKI_BEARER_TOKEN, or LOKI_USERNAME and
/// LOKI_PASSWORD for Grafana Cloud.
Loki {
/// Path to the import config. Its shape is documented in cli/src/experimental/logs/README.md.
#[arg(long)]
config: PathBuf,
/// Where to record progress. A resumed run re-sends only the shard it was inside.
#[arg(long, default_value = "loki-import.state")]
checkpoint: PathBuf,
/// Report the size, duration and mapping coverage of the run without sending anything.
#[arg(long, default_value_t = false)]
dry_run: bool,
/// Skip records older than the project's logs retention without asking. Required when no
/// terminal can answer the confirmation, for example in a Kubernetes Job.
#[arg(long, default_value_t = false)]
skip_expired: bool,
},
}
use std::io::IsTerminal;
use anyhow::{bail, Result};
use inquire::InquireError;
use super::config::LokiImportConfig;
use super::loki::{LokiAuth, LokiClient};
use super::mapping::Mapper;
use super::plan::{render, RunPlan};
use super::project::{check_login_matches_target, login_project_token};
use super::retention::{assess, project_retention_days, Expiry};
/// How many records a dry run pulls to prove the mapping. Large enough that a rule matching a
/// minority of records still shows a non-zero count, small enough to return in seconds.
const SAMPLE_RECORDS: usize = 100;
impl LogsCommand {
pub fn run(&self) -> Result<()> {
match self {
LogsCommand::Import { cmd } => cmd.run(),
}
}
}
impl ImportSource {
fn run(&self) -> Result<()> {
match self {
ImportSource::Loki {
config,
checkpoint,
dry_run,
skip_expired,
} => {
let text = std::fs::read_to_string(config).map_err(|error| {
anyhow::anyhow!("cannot read {}: {error}", config.display())
})?;
let parsed = LokiImportConfig::parse(&text)?;
let target_key = std::env::var(super::run::PROJECT_KEY_VAR)
.ok()
.filter(|key| !key.is_empty());
check_login_matches_target(
login_project_token().as_deref(),
target_key.as_deref(),
)?;
let retention_days = project_retention_days();
let expiry = assess(
parsed.range.from,
parsed.range.to,
retention_days,
chrono::Utc::now(),
);
if *dry_run {
return dry_run_report(&parsed, &expiry);
}
confirm_expiry(&expiry, *skip_expired)?;
run_import(&parsed, checkpoint, retention_days)
}
}
}
}
/// Intake drops records older than retention without telling the sender, so the run states what
/// it will drop and goes ahead only on an explicit yes.
fn confirm_expiry(expiry: &Expiry, skip_expired: bool) -> Result<()> {
let Some(confirmation) = expiry.confirmation() else {
return Ok(());
};
eprintln!("{}", confirmation.summary);
if skip_expired {
return Ok(());
}
// The prompt reads stdin, so stdin is what has to be a terminal. Output piped to a log file
// must not stop a run that can still be answered.
if !std::io::stdin().is_terminal() {
bail!("Stopped before sending anything. Pass --skip-expired to go ahead without asking.");
}
let confirmed = match inquire::Confirm::new(&confirmation.prompt)
.with_default(false)
.prompt()
{
Ok(answer) => answer,
Err(InquireError::OperationCanceled | InquireError::OperationInterrupted) => false,
Err(error) => return Err(error.into()),
};
if !confirmed {
bail!("Import cancelled. Nothing was sent.");
}
Ok(())
}
fn dry_run_report(config: &LokiImportConfig, expiry: &Expiry) -> Result<()> {
let client = LokiClient::new(
&config.source,
LokiAuth::from_env(),
reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()?,
);
let mapper = Mapper::new(config.extract.clone())?;
// Each selector gets an equal share of the sample. A first selector that filled the whole
// sample would report a field that only later selectors carry as NOT FOUND.
let per_selector = SAMPLE_RECORDS.div_ceil(config.range.select.len());
let from = expiry.effective_from(config.range.from, config.range.to);
let mut volume = 0;
let mut sample = Vec::new();
// A range wholly past retention leaves nothing to size or sample.
for selector in config
.range
.select
.iter()
.filter(|_| from < config.range.to)
{
volume += client.volume_bytes(selector, from, config.range.to)?;
let share = per_selector.min(SAMPLE_RECORDS - sample.len());
if share > 0 {
sample.extend(client.sample(selector, from, config.range.to, share)?);
}
}
let hits = mapper.hits(&sample);
let mapped: Vec<_> = sample.iter().map(|entry| mapper.map(entry)).collect();
// First record that actually carried each field. Reading only the first record would report
// NOT FOUND for a field most records have, in the output built to prevent exactly that.
let samples = [
(
"service_name",
mapped.iter().find_map(|r| r.service_name.clone()),
),
(
"severity",
mapped
.iter()
.find_map(|r| r.severity.as_ref().map(|(text, _)| text.clone())),
),
("trace_id", mapped.iter().find_map(|r| r.trace_id.clone())),
("span_id", mapped.iter().find_map(|r| r.span_id.clone())),
];
let sample_bytes: u64 = sample.iter().map(|entry| entry.line.len() as u64).sum();
let plan = RunPlan::build(config, from, volume, sample.len() as u64, sample_bytes);
print!("{}", render(config, &plan, &hits, &samples));
if let Some(confirmation) = expiry.confirmation() {
println!("\n{}", confirmation.summary);
}
Ok(())
}
fn run_import(
config: &LokiImportConfig,
checkpoint: &std::path::Path,
retention_days: Option<i64>,
) -> Result<()> {
use super::run::{intake_url, project_key_from_env, Importer};
let http = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()?;
let client = LokiClient::new(&config.source, LokiAuth::from_env(), http.clone());
let mapper = Mapper::new(config.extract.clone())?;
// The CLI already resolved the host from --host, then the stored login, then the default.
// Re-deriving it here would send an EU project's logs to US.
let host = crate::invocation_context::context().config.host.clone();
Importer {
config,
loki: &client,
mapper: &mapper,
http: &http,
intake_url: intake_url(&host, config.range.from),
project_key: project_key_from_env()?,
checkpoint_path: checkpoint,
retention_days,
}
.run()
}