Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
5.2 KB
Raw Blame
//! Karpathy "LLM Wiki" consolidation pipeline.
//!
//! M7a delivers the single-page variant: rewrite one
//! `sessions/<id>.md` page from raw observations via an LLM. The
//! store's sha256-equality short-circuit + supersession chain means
//! the rewrite is a *version*, not a destructive overwrite —
//! exactly the Karpathy pattern.
//!
//! M7b extends this to multi-page atomic fan-out.

pub mod auto_improve;
pub mod auto_improve_schedule;
pub mod auto_improve_telemetry;
pub mod bootstrap;
mod code_fence;
pub mod cold_cluster;
pub mod compaction;
pub mod consolidator;
pub mod curator;
pub mod dream;
pub mod embed;
pub mod entropy_filter;
pub mod experience;
pub mod keep_tokens;
mod link_guard;
pub mod lint;
mod path_sanitize;
pub mod profile;
pub mod projection;
mod prompt_template;
pub mod sweep;
pub mod types;

pub use auto_improve::{
    AutoImproveError, AutoImproveEvalConfig, AutoImproveEvidence, AutoImproveLlmResponse,
    AutoImproveProposal, AutoImproveRejectedCandidate, AutoImproveReport, AutoImproveReviewConfig,
    DEFAULT_AUTO_IMPROVE_MAX_CHANGED_CHARS_PER_PROPOSAL,
    DEFAULT_AUTO_IMPROVE_MAX_EDIT_CONTENT_CHARS, DEFAULT_AUTO_IMPROVE_MAX_EDITS_PER_PROPOSAL,
    DEFAULT_AUTO_IMPROVE_MAX_FINAL_BODY_CHARS, DEFAULT_AUTO_IMPROVE_MAX_INPUT_TOKENS,
    DEFAULT_AUTO_IMPROVE_MAX_PATCH_EDITS_PER_RUN, DEFAULT_AUTO_IMPROVE_MAX_PATCHABLE_BODY_CHARS,
    DEFAULT_AUTO_IMPROVE_MAX_PATCHABLE_PAGES, DEFAULT_AUTO_IMPROVE_MAX_PROCEDURE_PAGE_TOKENS,
    DEFAULT_AUTO_IMPROVE_MAX_PROPOSALS, DEFAULT_AUTO_IMPROVE_MAX_REJECTION_CONTEXT,
    DEFAULT_AUTO_IMPROVE_MAX_RULE_PAGE_TOKENS, DEFAULT_AUTO_IMPROVE_MIN_CONFIDENCE,
    DEFAULT_AUTO_IMPROVE_MIN_OBSERVATIONS, DEFAULT_AUTO_IMPROVE_MIN_SESSION_DURATION_SECS,
    DEFAULT_AUTO_IMPROVE_PATCHABLE_PAGE_PREFIXES, DEFAULT_AUTO_IMPROVE_PENDING_PATH,
    DEFAULT_AUTO_IMPROVE_PROPOSAL_ACTOR, DEFAULT_AUTO_IMPROVE_REJECTION_CONTEXT_DAYS,
    default_auto_improve_eval_targets, redacted_auto_improve_summary, run_auto_improve_review,
};
pub use auto_improve_schedule::{
    ScheduledAutoImproveSettings, ScheduledAutoImproveTickOutcome,
    initialize_auto_improve_scheduler_scopes, run_auto_improve_scheduler_tick,
};
pub use auto_improve_telemetry::{
    AutoImproveTelemetryFinding, AutoImproveTelemetryParams, AutoImproveTelemetryReport,
    AutoImproveTerminalRates, DEFAULT_AUTO_IMPROVE_TELEMETRY_SINCE_DAYS,
    DEFAULT_AUTO_IMPROVE_TELEMETRY_TOP_LIMIT, build_auto_improve_telemetry_report,
    render_auto_improve_telemetry_report_markdown, run_auto_improve_telemetry_report,
};
pub use bootstrap::{
    Bootstrap, BootstrapConfig, BootstrapError, BootstrapOutcome, BootstrapSource,
    DEFAULT_CHUNK_INPUT_TOKENS, ProjectNameStrategy, SourceCounts, SourceKind, collect_sources,
    derive_project_name, discover_main_repo_root, discover_repo_root, effective_chunk_budget,
    linked_worktree_main_root, path_is_git_ignored, plan_bootstrap_chunks, prune_sources_to_budget,
    read_identity_remotes,
};
pub use cold_cluster::{adaptive_eps, cosine_distance, dbscan};
pub use compaction::build_compacted_markdown;
pub use consolidator::{
    BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult,
    DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN, DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
    DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS, MIN_CONSOLIDATION_MAX_INPUT_TOKENS,
    MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request, disambiguate_colliding_session_title,
    effective_input_chars, existing_session_page_titles, redacted_error_summary,
};
pub use curator::{
    CuratorFinding, CuratorParams, CuratorReport, render_curator_report_markdown,
    run_curator_report, run_curator_report_with_breadth,
};
pub use dream::{
    ActivityClock, DEFAULT_DREAM_IDLE_WINDOW_SECS, DEFAULT_DREAM_MAX_CLUSTERS_PER_RUN,
    DEFAULT_DREAM_MIN_COLD_PAGES, DreamCancel, DreamConfig, DreamError, DreamMerge,
    DreamMergedPage, DreamReport, dream_idle_ready, run_dream_pass,
};
pub use embed::{
    EmbedBackfillCounts, EmbedBackfillError, EmbedBackfillOptions, run_embedding_backfill,
};
pub use entropy_filter::{EntropyFilterConfig, FilterVerdict, SkipReason, classify};
pub use experience::{EXPERIENCE_SYSTEM_PROMPT, ExperienceConfig, run_experience_review};
pub use keep_tokens::{mine_keep_tokens, mine_residue_tokens};
pub use lint::{
    DEFAULT_CONTRADICTION_SIM_HIGH, DEFAULT_CONTRADICTION_SIM_LOW, LintError, LintFinding,
    LintOptions, LintReport, run_lint, stale_days_for,
};
pub use sweep::{
    ColdClusterDedup, CompactedPage, DEFAULT_OBSERVATION_PRUNE_BATCH, EmbeddingCoord, EvictedPage,
    LedgerRetention, MergedCluster, ObservationRetention, SweepError, SweepReport, run_sweep,
    run_sweep_with_breadth, run_sweep_with_compaction, run_sweep_with_hygiene,
    run_sweep_with_ledgers, run_sweep_with_options,
};
pub use types::{
    ConsolidatedBatch, ConsolidatedPage, ConsolidatedPageUpdate, ConsolidationOutcome, PageKind,
    SlotKind,
};

// Integration tests compile into this crate's test harness instead of a
// separate binary: every test binary is another link and, on macOS and
// Windows, another first-run malware scan. They still exercise only the
// public API; `extern crate self` lets them keep addressing it by crate name.
#[cfg(test)]
extern crate self as ai_memory_consolidate;
#[cfg(test)]
#[path = "../tests/suite/mod.rs"]
mod integration;